Python Signature.from_dict方法代码示例

# 需要导入模块: from celery.canvas import Signature [as 别名]
# 或者: from celery.canvas.Signature import from_dict [as 别名]
def setup_cluster(task, *args, **kwargs):
    cluster = kwargs['cluster']

    if '_id' in cluster:
        task.taskflow.logger.info(
            'We are using an existing cluster: %s' % cluster['name'])
    else:
        task.taskflow.logger.info('We are creating an EC2 cluster.')
        task.logger.info('Cluster name %s' % cluster['name'])
        kwargs['machine'] = cluster.get('machine')
        profile = kwargs.get('profile')
        cluster = create_ec2_cluster(
            task, cluster, profile, kwargs['image_spec'])
        task.logger.info('Cluster started.')

    # Call any follow on task
    if 'next' in kwargs:
        kwargs['cluster'] = cluster
        next = Signature.from_dict(kwargs['next'])

        if next.task == 'celery.chain':
            # If we are dealing with a chain we want to update the arg and
            # kwargs passed into the chain.
            first_task = next.kwargs['tasks'][0]
            if first_task:
                if args:
                    first_task.args = tuple(args) + tuple(first_task.args)

                if kwargs:
                    first_task.kwargs = dict(first_task.kwargs, **kwargs)

        next.delay(*args, **kwargs)

# 需要导入模块: from celery.canvas import Signature [as 别名]
# 或者: from celery.canvas.Signature import from_dict [as 别名]
def postcommit_after_request(response, base_status_error_code=500):
    if response.status_code >= base_status_error_code:
        _local.postcommit_queue = OrderedDict()
        _local.postcommit_celery_queue = OrderedDict()
        return response
    try:
        if postcommit_queue():
            number_of_threads = 30  # one db connection per greenlet, let's share
            pool = Pool(number_of_threads)
            for func in postcommit_queue().values():
                pool.spawn(func)
            pool.join(timeout=5.0, raise_error=True)  # 5 second timeout and reraise exceptions

        if postcommit_celery_queue():
            if settings.USE_CELERY:
                for task_dict in postcommit_celery_queue().values():
                    task = Signature.from_dict(task_dict)
                    task.apply_async()
            else:
                for task in postcommit_celery_queue().values():
                    task()

    except AttributeError as ex:
        if not settings.DEBUG_MODE:
            logger.error('Post commit task queue not initialized: {}'.format(ex))
    return response

# 需要导入模块: from celery.canvas import Signature [as 别名]
# 或者: from celery.canvas.Signature import from_dict [as 别名]
def setup_cluster(task, *args,**kwargs):
    cluster = kwargs['cluster']

    if '_id' in cluster:
        task.taskflow.logger.info('We are using an existing cluster: %s' % cluster['name'])
    else:
        task.taskflow.logger.info('We are creating an EC2 cluster.')
        task.logger.info('Cluster name %s' % cluster['name'])
        kwargs['machine'] = cluster.get('machine')
        ami = kwargs.get('ami')
        profile = kwargs.get('profile')
        cluster = create_ec2_cluster(task, cluster, profile, ami)
        task.logger.info('Cluster started.')

    # Call any follow on task
    if 'next' in kwargs:
        kwargs['cluster'] = cluster
        next = Signature.from_dict(kwargs['next'])
        next.delay(*args, **kwargs)

# 需要导入模块: from celery.canvas import Signature [as 别名]
# 或者: from celery.canvas.Signature import from_dict [as 别名]
 def inner(*args, **kwargs):
     taskset = kwargs.pop("_taskset", None)
     rv = f(*args, **kwargs)
     if taskset is not None:
         done = False
         with transaction.atomic():
             taskset_id = taskset["taskset_id"]
             sync_row = TaskSetMeta.objects.select_for_update().filter(
                 id=taskset_id).all()
             if sync_row:
                 assert len(sync_row) == 1
                 sync_row = sync_row[0]
                 sync_row.count -= 1
                 sync_row.save()
                 if sync_row.count == 0:
                     logger.info("Finished taskset id %i" % taskset_id)
                     done = True
                 else:
                     logger.info("Taskset %i has %i tasks remaining" % (taskset_id, sync_row.count))
         if done:
             callback = Signature.from_dict(taskset["callback"])
             callback.apply_async()
     return rv

# 需要导入模块: from celery.canvas import Signature [as 别名]
# 或者: from celery.canvas.Signature import from_dict [as 别名]
def postcommit_celery_task_wrapper(queue):
    # chain.apply calls the tasks synchronously without re-enqueuing each one
    # http://stackoverflow.com/questions/34177131/how-to-solve-python-celery-error-when-using-chain-encodeerrorruntimeerrormaxi?answertab=votes#tab-top
    # celery serialized signatures into dictionaries, so we need to deserialize here
    # https://sentry.cos.io/sentry/osf-iy/issues/289209/
    chain([Signature.from_dict(task_dict) for task_dict in queue.values()]).apply()