Python celery.Task方法代码示例

# 需要导入模块: import celery [as 别名]
# 或者: from celery import Task [as 别名]
def bind(self, app=None):
        """Intercept binding of task to (celery) app

        Here we take the half-finished generated Task class and
        replace the async run method with a sync run method that
        executes the original method inside the asyncio loop.
        """
        if asyncio.iscoroutinefunction(self.run):  # only for async funcs
            @wraps(self.run)
            def sync_run(*args, **kwargs):
                largs = list(args)  # need list so that pre-run can modify
                self.loop.run_until_complete(self.async_pre_run(largs, kwargs))
                return self.loop.run_until_complete(self._async_run(*largs, **kwargs))

            # swap run method with wrapper defined above
            self._async_run, self.run = self.run, sync_run

            if not self.loop.is_running():
                self.loop.run_until_complete(self.async_init())
        super().bind(app) 

# 需要导入模块: import celery [as 别名]
# 或者: from celery import Task [as 别名]
def fetch_celery_task_state(async_result: AsyncResult) -> \
        Tuple[str, Union[str, ExceptionWithTraceback], Any]:
    """
    Fetch and return the state of the given celery task. The scope of this function is
    global so that it can be called by subprocesses in the pool.

    :param async_result: a tuple of the Celery task key and the async Celery object used
        to fetch the task's state
    :type async_result: tuple(str, celery.result.AsyncResult)
    :return: a tuple of the Celery task key and the Celery state and the celery info
        of the task
    :rtype: tuple[str, str, str]
    """

    try:
        with timeout(seconds=OPERATION_TIMEOUT):
            # Accessing state property of celery task will make actual network request
            # to get the current state of the task
            info = async_result.info if hasattr(async_result, 'info') else None
            return async_result.task_id, async_result.state, info
    except Exception as e:  # pylint: disable=broad-except
        exception_traceback = f"Celery Task ID: {async_result}\n{traceback.format_exc()}"
        return async_result.task_id, ExceptionWithTraceback(e, exception_traceback), None 

# 需要导入模块: import celery [as 别名]
# 或者: from celery import Task [as 别名]
def after_return(self, status, retval, task_id, args, kwargs, einfo):
        """
        Handler called after the task returns.
        :param status: Current task state.
        :param retval: Task return value/exception.
        :param task_id: Unique id of the task.
        :param args: Original arguments for the task that returned.
        :param kwargs: Original keyword arguments for the task that returned.
        :param einfo: ExceptionInfo instance, containing the traceback (if any).
        :return: None
        """
        logger.debug(
            "In %s.after_return: %s, %s, %s, %s."
            % (self.__class__.__name__, status, retval, task_id, einfo)
        )
        self.__decrement_request_tags()
        super(WebSightBaseTask, self).after_return(status, retval, task_id, args, kwargs, einfo) 

# 需要导入模块: import celery [as 别名]
# 或者: from celery import Task [as 别名]
def after_return(self, status, retval, task_id, args, kwargs, einfo):
        self.logger.debug('Task %s("%s") returned %s. Result: """%s"""', self.name, args, status, retval)
        meta = kwargs.get('meta', {})
        nolog = meta.get('nolog', False)

        # In case of emergency log this task
        if not nolog and not self.all_done:
            if isinstance(retval, dict):
                result = retval.copy()
            else:
                if einfo:
                    result = {'detail': str(einfo.exception)}
                else:
                    result = {'detail': str(retval)}

            if 'meta' not in result:
                result['meta'] = meta

            # noinspection PyUnresolvedReferences
            result['meta']['cb_name'] = LOGTASK
            meta['task_status'] = status
            meta['cleanup'] = True
            t = send_task_forever(task_id, LOGTASK, nolog=nolog, args=(result, task_id), kwargs=meta,
                                  queue=Q_MGMT, expires=None, task_id=task_id_from_task_id(task_id))
            self.logger.warn('Created emergency log task %s', t.id) 

# 需要导入模块: import celery [as 别名]
# 或者: from celery import Task [as 别名]
def on_failure(self, exc, task_id, args, kwargs, einfo):
        '''
        When a task fails, update the task DB with a "Failed"
        status. Dump a traceback to local logs
        '''
        logger.error('Task #{} failed'.format(args[2]))
        logger.error('Traceback info:\n{}'.format(einfo))

        # Initialize the connection to the task DB
        db.init_db()

        scan_time = datetime.now().isoformat()

        # Update the task DB with the failure
        db.update_task(
            task_id=args[2],
            task_status='Failed',
            timestamp=scan_time,
        ) 

# 需要导入模块: import celery [as 别名]
# 或者: from celery import Task [as 别名]
def on_failure(self, exc, task_id, args, kwargs, einfo):
        """Log exceptions when a celery task fails."""
        LOGGER.exception("Task failed: %s", exc, exc_info=exc)
        super().on_failure(exc, task_id, args, kwargs, einfo) 

# 需要导入模块: import celery [as 别名]
# 或者: from celery import Task [as 别名]
def _get_call_args(self, args, kwargs):
        call_args = self._signature.bind(*args, **kwargs).arguments
        # Remove the task instance from the kwargs. This only happens when the
        # task has the 'bind' attribute set to True. We remove it, as the task
        # has a memory pointer in its repr, that will change between the task
        # caller and the celery worker
        if isinstance(call_args.get('self'), Task):
            del call_args['self']
        return call_args 

# 需要导入模块: import celery [as 别名]
# 或者: from celery import Task [as 别名]
def set_task_status(analysis_pk, task_status):
    try:
        from .models import Analysis
        analysis = Analysis.objects.get(pk=analysis_pk)
        analysis.status = task_status
        analysis.save(update_fields=["status"])
        logger.info('Task Status Update: analysis_pk: {}, status: {}'.format(analysis_pk, task_status))
    except Exception as e:
        logger.error('Task Status Update: Failed')
        logger.exception(str(e)) 

# 需要导入模块: import celery [as 别名]
# 或者: from celery import Task [as 别名]
def __init__(self, exception: Exception, exception_traceback: str):
        self.exception = exception
        self.traceback = exception_traceback


# Task instance that is sent over Celery queues
# TaskInstanceKeyType, SimpleTaskInstance, Command, queue_name, CallableTask 

# 需要导入模块: import celery [as 别名]
# 或者: from celery import Task [as 别名]
def send_task_to_executor(task_tuple: TaskInstanceInCelery) \
        -> Tuple[TaskInstanceKeyType, CommandType, Union[AsyncResult, ExceptionWithTraceback]]:
    """Sends task to executor."""
    key, _, command, queue, task_to_run = task_tuple
    try:
        with timeout(seconds=OPERATION_TIMEOUT):
            result = task_to_run.apply_async(args=[command], queue=queue)
    except Exception as e:  # pylint: disable=broad-except
        exception_traceback = "Celery Task ID: {}\n{}".format(key, traceback.format_exc())
        result = ExceptionWithTraceback(e, exception_traceback)

    return key, command, result 

# 需要导入模块: import celery [as 别名]
# 或者: from celery import Task [as 别名]
def test_export_task(self):
        assert isinstance(export, Task)
        assert export.name == 'export.project'
        kwargs = {
            'org_slug': 'my-org',
            'project_slug': 'my-proj',
            'api_key': 'a-key',
            'output_type': 'all',
        }
        export(**kwargs) 

# 需要导入模块: import celery [as 别名]
# 或者: from celery import Task [as 别名]
def __call__(self, *args, **kwargs):
        try:
            _t_args = _walk_obj(args, self._maybe_transform_argument)
            _t_kwargs = _walk_obj(kwargs, self._maybe_transform_argument)

            results = super(Task, self).__call__(*_t_args, **_t_kwargs)

            if hasattr(self.request, 'girder_result_hooks'):
                if isinstance(results, tuple):
                    results = tuple([self._maybe_transform_result(i, r)
                                     for i, r in enumerate(results)])
                else:
                    results = self._maybe_transform_result(0, results)

            return results
        except Exception:
            if hasattr(self.request, 'girder_result_hooks'):
                for hook in self.request.girder_result_hooks:
                    hook.exception()
            raise
        finally:
            _walk_obj(args, self._maybe_cleanup)
            _walk_obj(kwargs, self._maybe_cleanup)
            if hasattr(self.request, 'girder_result_hooks'):
                for hook in self.request.girder_result_hooks:
                    self._maybe_cleanup(hook) 

# 需要导入模块: import celery [as 别名]
# 或者: from celery import Task [as 别名]
def process(self, query, options=None):
        self.query = query
        self.options = options or {}

        if isinstance(self.query, str):
            self.query = {'number': self.query}

        logger.info('Starting download process for query=%s, options=%s', query, options)
        self.result = {'metadata': {'query': self.query, 'options': self.options, 'files': []}}

        # Download and store the document
        try:
            self.download()
            self.enrich()
            self.store()

        # If this is an unrecoverable error, we reject it so that it's redelivered
        # to the dead letter exchange and we can manually inspect the situation.
        # http://docs.celeryproject.org/en/latest/userguide/tasks.html#reject
        except (NoResults, UnknownDocumentType) as ex:
            # FIXME: Task keeps being in PENDING state when rejected
            #raise celery.exceptions.Reject(reason=ex, requeue=False)

            # Workaround: Just re-raise the original exception
            raise

        # Otherwise, let's retry again after some time
        except Exception as ex:
            raise self.retry(exc=ex, countdown=self.RETRY_COUNTDOWN, max_retries=self.RETRY_ATTEMPTS)

        return self.result

    # TODO: Implement "after_return" and "on_retry"
    # # http://docs.celeryproject.org/en/latest/userguide/tasks.html#handlers 

# 需要导入模块: import celery [as 别名]
# 或者: from celery import Task [as 别名]
def poll_task(self):
        logger.info('Polling task with id=%s', self.task.id)
        try:
            while not self.task.ready():
                logger.info('Task with id=%s in state %s', self.task.id, self.task.state)
                time.sleep(1)

            result = self.task.get()
            logger.info('Download ready')
            return result

        except Exception as ex:
            logger.error('Download failed with exception: "%s: %s"', ex.__class__.__name__, ex)
            raise 

# 需要导入模块: import celery [as 别名]
# 或者: from celery import Task [as 别名]
def poll_group(self):
        results = OrderedDict()
        errors = []
        while self.task.results:

            for subtask in self.task.results:

                if subtask.ready():
                    logger.info('Task with id=%s in state %s.', subtask.id, subtask.state)

                    try:
                        result = subtask.get()
                        logger.info('Download succeeded')

                        if result['metadata']['options'].get('use-application-id'):
                            key = result['metadata']['document_identifiers']['application']
                        else:
                            key = result['metadata']['query']['number']

                        results[key] = result

                    except Exception as ex:
                        logger.error('Download failed with exception: "%s: %s"', ex.__class__.__name__, ex)
                        error = {
                            'message': '{}: {}'.format(ex.__class__.__name__, ex)
                        }
                        if hasattr(ex, 'more_info'):
                            error.update(ex.more_info)
                        errors.append(error)

                    self.task.results.remove(subtask)

                else:
                    logger.info('Task %s in state %s. Metadata: %s', subtask.id, subtask.state, subtask.info)

            time.sleep(1)

        response = OrderedDict()
        response['results'] = results
        response['errors']  = errors
        return response