Python canvas.Signature类代码示例

 def test_link_on_scalar(self):
     x = Signature('TASK', link=Signature('B'))
     self.assertTrue(x.options['link'])
     x.link(Signature('C'))
     self.assertIsInstance(x.options['link'], list)
     self.assertIn(Signature('B'), x.options['link'])
     self.assertIn(Signature('C'), x.options['link'])

 def test_link_on_scalar(self):
     x = Signature('TASK', link=Signature('B'))
     assert x.options['link']
     x.link(Signature('C'))
     assert isinstance(x.options['link'], list)
     assert Signature('B') in x.options['link']
     assert Signature('C') in x.options['link']

 def test_replace(self):
     x = Signature('TASK', ('A'), {})
     self.assertTupleEqual(x.replace(args=('B', )).args, ('B', ))
     self.assertDictEqual(x.replace(kwargs={'FOO': 'BAR'}).kwargs,
             {'FOO': 'BAR'})
     self.assertDictEqual(x.replace(options={'task_id': '123'}).options,
             {'task_id': '123'})

 def test_replace(self):
     x = Signature('TASK', ('A'), {})
     assert x.replace(args=('B',)).args == ('B',)
     assert x.replace(kwargs={'FOO': 'BAR'}).kwargs == {
         'FOO': 'BAR',
     }
     assert x.replace(options={'task_id': '123'}).options == {
         'task_id': '123',
     }

def postcommit_after_request(response, base_status_error_code=500):
    if response.status_code >= base_status_error_code:
        _local.postcommit_queue = OrderedDict()
        _local.postcommit_celery_queue = OrderedDict()
        return response
    try:
        if postcommit_queue():
            number_of_threads = 30  # one db connection per greenlet, let's share
            pool = Pool(number_of_threads)
            for func in postcommit_queue().values():
                pool.spawn(func)
            pool.join(timeout=5.0, raise_error=True)  # 5 second timeout and reraise exceptions

        if postcommit_celery_queue():
            if settings.USE_CELERY:
                for task_dict in postcommit_celery_queue().values():
                    task = Signature.from_dict(task_dict)
                    task.apply_async()
            else:
                for task in postcommit_celery_queue().values():
                    task()

    except AttributeError as ex:
        if not settings.DEBUG_MODE:
            logger.error('Post commit task queue not initialized: {}'.format(ex))
    return response

        def prepare_steps(self, args, tasks):
            steps = deque(tasks)
            next_step = prev_task = prev_res = None
            tasks, results = [], []
            i = 0
            while steps:
                # First task get partial args from chain.
                task = maybe_subtask(steps.popleft())
                task = task.clone() if i else task.clone(args)
                res = task.freeze()
                i += 1

                if isinstance(task, group):
                    # automatically upgrade group(..) | s to chord(group, s)
                    try:
                        next_step = steps.popleft()
                        # for chords we freeze by pretending it's a normal
                        # task instead of a group.
                        res = Signature.freeze(task)
                        task = chord(task, body=next_step, task_id=res.task_id)
                    except IndexError:
                        pass  # no callback, so keep as group
                if prev_task:
                    # link previous task to this task.
                    prev_task.link(task)
                    # set the results parent attribute.
                    res.parent = prev_res

                if not isinstance(prev_task, chord):
                    results.append(res)
                    tasks.append(task)
                prev_task, prev_res = task, res

            return tasks, results

def setup_cluster(task, *args, **kwargs):
    cluster = kwargs['cluster']

    if '_id' in cluster:
        task.taskflow.logger.info(
            'We are using an existing cluster: %s' % cluster['name'])
    else:
        task.taskflow.logger.info('We are creating an EC2 cluster.')
        task.logger.info('Cluster name %s' % cluster['name'])
        kwargs['machine'] = cluster.get('machine')
        profile = kwargs.get('profile')
        cluster = create_ec2_cluster(
            task, cluster, profile, kwargs['image_spec'])
        task.logger.info('Cluster started.')

    # Call any follow on task
    if 'next' in kwargs:
        kwargs['cluster'] = cluster
        next = Signature.from_dict(kwargs['next'])

        if next.task == 'celery.chain':
            # If we are dealing with a chain we want to update the arg and
            # kwargs passed into the chain.
            first_task = next.kwargs['tasks'][0]
            if first_task:
                if args:
                    first_task.args = tuple(args) + tuple(first_task.args)

                if kwargs:
                    first_task.kwargs = dict(first_task.kwargs, **kwargs)

        next.delay(*args, **kwargs)

        def prepare_steps(self, args, tasks):
            app = self.app
            steps = deque(tasks)
            next_step = prev_task = prev_res = None
            tasks, results = [], []
            i = 0
            while steps:
                # First task get partial args from chain.
                task = maybe_signature(steps.popleft(), app=app)
                task = task.clone() if i else task.clone(args)
                res = task.freeze()
                i += 1

                if isinstance(task, group):
                    task = maybe_unroll_group(task)
                if isinstance(task, chain):
                    # splice the chain
                    steps.extendleft(reversed(task.tasks))
                    continue

                elif isinstance(task, group) and steps and \
                        not isinstance(steps[0], group):
                    # automatically upgrade group(..) | s to chord(group, s)
                    try:
                        next_step = steps.popleft()
                        # for chords we freeze by pretending it's a normal
                        # task instead of a group.
                        res = Signature.freeze(next_step)
                        task = chord(task, body=next_step, task_id=res.task_id)
                    except IndexError:
                        pass  # no callback, so keep as group
                if prev_task:
                    # link previous task to this task.
                    prev_task.link(task)
                    # set the results parent attribute.
                    if not res.parent:
                        res.parent = prev_res

                if not isinstance(prev_task, chord):
                    results.append(res)
                    tasks.append(task)
                prev_task, prev_res = task, res

            print(tasks)

            return tasks, results

def setup_cluster(task, *args,**kwargs):
    cluster = kwargs['cluster']

    if '_id' in cluster:
        task.taskflow.logger.info('We are using an existing cluster: %s' % cluster['name'])
    else:
        task.taskflow.logger.info('We are creating an EC2 cluster.')
        task.logger.info('Cluster name %s' % cluster['name'])
        kwargs['machine'] = cluster.get('machine')
        ami = kwargs.get('ami')
        profile = kwargs.get('profile')
        cluster = create_ec2_cluster(task, cluster, profile, ami)
        task.logger.info('Cluster started.')

    # Call any follow on task
    if 'next' in kwargs:
        kwargs['cluster'] = cluster
        next = Signature.from_dict(kwargs['next'])
        next.delay(*args, **kwargs)

 def inner(*args, **kwargs):
     taskset = kwargs.pop("_taskset", None)
     rv = f(*args, **kwargs)
     if taskset is not None:
         done = False
         with transaction.atomic():
             taskset_id = taskset["taskset_id"]
             sync_row = TaskSetMeta.objects.select_for_update().filter(
                 id=taskset_id).all()
             if sync_row:
                 assert len(sync_row) == 1
                 sync_row = sync_row[0]
                 sync_row.count -= 1
                 sync_row.save()
                 if sync_row.count == 0:
                     logger.info("Finished taskset id %i" % taskset_id)
                     done = True
                 else:
                     logger.info("Taskset %i has %i tasks remaining" % (taskset_id, sync_row.count))
         if done:
             callback = Signature.from_dict(taskset["callback"])
             callback.apply_async()
     return rv

def postcommit_celery_task_wrapper(queue):
    # chain.apply calls the tasks synchronously without re-enqueuing each one
    # http://stackoverflow.com/questions/34177131/how-to-solve-python-celery-error-when-using-chain-encodeerrorruntimeerrormaxi?answertab=votes#tab-top
    # celery serialized signatures into dictionaries, so we need to deserialize here
    # https://sentry.cos.io/sentry/osf-iy/issues/289209/
    chain([Signature.from_dict(task_dict) for task_dict in queue.values()]).apply()

 def test_reduce(self):
     x = Signature('TASK', (2, 4), app=self.app)
     fun, args = x.__reduce__()
     assert fun(*args) == x

 def test_json(self):
     x = Signature('TASK', link=Signature('B', app=self.app), app=self.app)
     assert x.__json__() == dict(x)

 def test_call(self):
     x = Signature('foo', (1, 2), {'arg1': 33}, app=self.app)
     x.type = Mock(name='type')
     x(3, 4, arg2=66)
     x.type.assert_called_with(3, 4, 1, 2, arg1=33, arg2=66)

 def test_json(self):
     x = Signature('TASK', link=Signature('B', app=self.app), app=self.app)
     self.assertDictEqual(x.__json__(), dict(x))