https://github.com/SoftwareHeritage/swh-lister
Raw File
Tip revision: 4b27f9d9c4076d3b2aa4e6e6903a41ec7967d724 authored by Archit Agrawal on 24 March 2019, 05:24:12 UTC
Updated toplevel function names in README
Tip revision: 4b27f9d
tasks.py
# Copyright (C) 2018 the Software Heritage developers
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information

import random

from celery import group
from swh.scheduler.celery_backend.config import app

from .. import utils
from .lister import GitLabLister


NBPAGES = 10


def new_lister(api_baseurl='https://gitlab.com/api/v4',
               instance=None, sort='asc', per_page=20):
    return GitLabLister(
        api_baseurl=api_baseurl, instance=instance, sort=sort,
        per_page=per_page)


@app.task(name=__name__ + '.IncrementalGitLabLister')
def incremental_gitlab_lister(**lister_args):
    lister_args['sort'] = 'desc'
    lister = new_lister(**lister_args)
    total_pages = lister.get_pages_information()[1]
    # stopping as soon as existing origins for that instance are detected
    lister.run(min_bound=1, max_bound=total_pages, check_existence=True)


@app.task(name=__name__ + '.RangeGitLabLister')
def range_gitlab_lister(start, end, **lister_args):
    lister = new_lister(**lister_args)
    lister.run(min_bound=start, max_bound=end)


@app.task(name=__name__ + '.FullGitLabRelister', bind=True)
def full_gitlab_relister(self, **lister_args):
    lister = new_lister(**lister_args)
    _, total_pages, _ = lister.get_pages_information()
    ranges = list(utils.split_range(total_pages, NBPAGES))
    random.shuffle(ranges)
    promise = group(range_gitlab_lister.s(minv, maxv, **lister_args)
                    for minv, maxv in ranges)()
    self.log.debug('%s OK (spawned %s subtasks)' % (self.name, len(ranges)))
    try:
        promise.save()
    except NotImplementedError:
        self.log.info('Unable to call save_group with current result backend.')
    return promise.id


@app.task(name=__name__ + '.ping')
def ping():
    return 'OK'
back to top