Python rq.Queue方法代码示例

# 需要导入模块: import rq [as 别名]
# 或者: from rq import Queue [as 别名]
def get_status(task_id):
    with Connection(redis.from_url(current_app.config["REDIS_URL"])):
        q = Queue()
        task = q.fetch_job(task_id)
    if task:
        response_object = {
            "status": "success",
            "data": {
                "task_id": task.get_id(),
                "task_status": task.get_status(),
                "task_result": task.result,
            },
        }
    else:
        response_object = {"status": "error"}
    return jsonify(response_object) 

# 需要导入模块: import rq [as 别名]
# 或者: from rq import Queue [as 别名]
def test_basic(sentry_init, capture_events):
    sentry_init(integrations=[RqIntegration()])
    events = capture_events()

    queue = rq.Queue(connection=FakeStrictRedis())
    worker = rq.SimpleWorker([queue], connection=queue.connection)

    queue.enqueue(crashing_job, foo=42)
    worker.work(burst=True)

    (event,) = events

    (exception,) = event["exception"]["values"]
    assert exception["type"] == "ZeroDivisionError"
    assert exception["mechanism"]["type"] == "rq"
    assert exception["stacktrace"]["frames"][-1]["vars"]["foo"] == "42"

    assert event["transaction"] == "tests.integrations.rq.test_rq.crashing_job"
    assert event["extra"]["rq-job"] == {
        "args": [],
        "description": "tests.integrations.rq.test_rq.crashing_job(foo=42)",
        "func": "tests.integrations.rq.test_rq.crashing_job",
        "job_id": event["extra"]["rq-job"]["job_id"],
        "kwargs": {"foo": 42},
    } 

# 需要导入模块: import rq [as 别名]
# 或者: from rq import Queue [as 别名]
def test_transport_shutdown(sentry_init, capture_events_forksafe):
    sentry_init(integrations=[RqIntegration()])

    events = capture_events_forksafe()

    queue = rq.Queue(connection=FakeStrictRedis())
    worker = rq.Worker([queue], connection=queue.connection)

    queue.enqueue(crashing_job, foo=42)
    worker.work(burst=True)

    event = events.read_event()
    events.read_flush()

    (exception,) = event["exception"]["values"]
    assert exception["type"] == "ZeroDivisionError" 

# 需要导入模块: import rq [as 别名]
# 或者: from rq import Queue [as 别名]
def create_app():

    # setting the static_url_path to blank serves static files from the web root
    app = Flask(__name__, static_url_path='')
    app.config.from_object(__name__)

    Swagger(app, template_file='definitions.yaml')

    app.redis = Redis.from_url(app.config['REDIS_URL'])
    app.task_queue = rq.Queue('recon-tasks', connection=app.redis)

    @app.after_request
    def disable_cache(response):
        response.headers['Cache-Control'] = 'no-cache, no-store, must-revalidate'
        return response

    @app.route('/')
    def index():
        return render_template('index.html', workspaces=recon._get_workspaces())

    from recon.core.web.api import resources
    app.register_blueprint(resources)

    return app 

# 需要导入模块: import rq [as 别名]
# 或者: from rq import Queue [as 别名]
def test_job_archiving_not_supported(self):
        """Check if it fails when archiving is not supported"""

        backend_args = {
            'uri': 'http://example.com/',
            'gitpath': os.path.join(self.dir, 'data/git_log.txt')
        }
        archive_args = {
            'archive_path': self.tmp_path,
            'fetch_from_archive': True
        }

        q = rq.Queue('queue', is_async=False)  # noqa: W606

        with self.assertRaises(AttributeError):
            _ = q.enqueue(execute_perceval_job,
                          backend='git', backend_args=backend_args,
                          category='commit',
                          qitems='items', task_id='mytask', job_number=8,
                          archive_args=archive_args) 

# 需要导入模块: import rq [as 别名]
# 或者: from rq import Queue [as 别名]
def __init__(self, registry, conn, queues, polling=SCHED_POLLING, async_mode=True):
        super().__init__()
        self.registry = registry
        self.conn = conn
        self.polling = polling
        self.async_mode = async_mode

        self._rwlock = RWLock()
        self._delayer = sched.scheduler()
        self._queues = {
            queue_id: rq.Queue(queue_id,
                               connection=self.conn,
                               is_async=self.async_mode)  # noqa: W606
            for queue_id in queues
        }
        self._tasks_events = {}
        self._tasks_jobs = {} 

# 需要导入模块: import rq [as 别名]
# 或者: from rq import Queue [as 别名]
def get_worker(self, listen=()):
        if not listen:
            listen = self.config["queues"]

        def send_to_sentry(job, *exc_info):
            self.sentry.captureException(
                exc_info=exc_info,
                extra={
                    "job_id": job.id,
                    "func": job.__name__,
                    "args": job.args,
                    "kwargs": job.kwargs,
                    "description": job.description,
                },
            )

        exception_handlers = [send_to_sentry]

        return Worker(
            [QueueType(k, connection=self.connection) for k in listen],
            exception_handlers=exception_handlers,
            connection=self.connection,
        ) 

# 需要导入模块: import rq [as 别名]
# 或者: from rq import Queue [as 别名]
def test_set_rq():

    """
    When Redis params are provided, set an RQ instance.
    """

    config = get_config('read/set-rq')

    # Should set an instance.
    assert isinstance(config.rq, Queue)

    args = config.rq.connection.connection_pool.connection_kwargs

    # Should use config args.
    assert args['host'] == 'host'
    assert args['port'] == 1337
    assert args['db']   == 1 

# 需要导入模块: import rq [as 别名]
# 或者: from rq import Queue [as 别名]
def _calc_rq_free() -> Dict[str, Any]:
    """Parses the output of `rq info` to return total number
    of workers and the count of workers currently idle."""

    conn = default_redis_conn()
    with rq.Connection(connection=conn):
        workers: List[rq.Worker] = [w for w in rq.Worker.all()]
    idle_workers = [w for w in workers if w.get_state() == 'idle']
    resp = {
        'workersTotal': len(workers),
        'workersIdle': len(idle_workers),
        'workersUnknown': len([w for w in workers if w.get_state() == '?'])
    }
    queues = 'default', 'build', 'publish'
    resp.update({f'queue{q.capitalize()}Size':
                 len(rq.Queue(f'gigantum-{q}-queue', connection=conn)) for q in queues})
    return resp 

# 需要导入模块: import rq [as 别名]
# 或者: from rq import Queue [as 别名]
def get_tasks_on_queue(queue_name):
    q = Queue(queue_name, connection=Redis(host=envget('redis.host')))
    jobs = q.jobs
    tasks = []
    for job in jobs:
        task = {"date_enqueued": str(
            process_date(job.to_dict().get('enqueued_at')))}
        '''
        to_dict() returns something like this:
        {u'origin': u'task_no_vt', u'status': u'queued', u'description': u"Api.task.generic_task('N7UFZ56FQDITJ34F40TZB50XAWVNW575QGIL4YEC')", u'created_at': '2017-03-03T20:14:47Z', u'enqueued_at': '2017-03-03T20:14:47Z', u'timeout': 31536000, u'data': '\x80\x02(X\x15\x00\x00\x00Api.task.generic_taskq\x01NU(N7UFZ56FQDITJ34F40TZB50XAWVNW575QGIL4YECq\x02\x85q\x03}q\x04tq\x05.'}
        '''
        task_id = re.search('[A-Z0-9]{40}', job.to_dict().get('description'))
        if task_id is None:
            continue
        task['task_id'] = task_id.group(0)
        task['hashes'] = count_valid_hashes_in_task(task['task_id'])
        tasks.append(task)
    return tasks 

# 需要导入模块: import rq [as 别名]
# 或者: from rq import Queue [as 别名]
def add_task(requested):
    task_id = id_generator(40)
    if requested.get('document_name') is None:
        requested["document_name"] = ""

    response = {"requested": requested,
                "date_enqueued": datetime.datetime.now(),
                "task_id": task_id}
    save(response)
    if requested.get('vt_samples'):
        queue_name = "task_private_vt"  # task needs a private VT api
    elif requested.get('vt_av') and not requested.get('vt_samples'):
        queue_name = "task_public_vt"  # task needs a public VT api
    else:
        queue_name = "task_no_vt"  # task doesn't need VT
    q = Queue(queue_name, connection=Redis(host=envget('redis.host')))
    job = q.enqueue('Api.task.generic_task', args=(task_id,), timeout=31536000)
    return task_id 

# 需要导入模块: import rq [as 别名]
# 或者: from rq import Queue [as 别名]
def app_with_scout(redis_conn, scout_config=None):
    """
    Context manager that configures a Huey app with Scout installed.
    """
    # Enable Scout by default in tests.
    if scout_config is None:
        scout_config = {}
    scout_config.setdefault("monitor", True)
    scout_config["core_agent_launch"] = False

    # Reset global state
    scout_apm.rq.install_attempted = False
    scout_apm.rq.installed = None

    # Setup according to https://docs.scoutapm.com/#rq
    # Using job_class argument to Queue
    Config.set(**scout_config)
    queue = Queue(name="myqueue", connection=redis_conn)
    worker = scout_apm.rq.SimpleWorker([queue], connection=queue.connection)

    App = namedtuple("App", ["queue", "worker"])
    try:
        yield App(queue=queue, worker=worker)
    finally:
        Config.reset_all() 

# 需要导入模块: import rq [as 别名]
# 或者: from rq import Queue [as 别名]
def request_poller(queue_1, queue_2, nb_to_do):
    requests = pollout_requests(queue_2, nb_to_do)
    try:
        with Connection(Redis()) as conn:
            q = Queue(queue_1, connection=conn)
    except:
        mod.display("ASYNC_HTTP",
                    "ERROR",
                    "Could not establish connection with Redis, check if you have redis_host, \
                    redis_port and maybe redis_password in /config/config.ini")

    loop = asyncio.get_event_loop()
    future = asyncio.ensure_future(run(requests))
    x = loop.run_until_complete(future)
    loop.close()
    for y in x:
        if y is not None:
            try:
                q.enqueue(module_worker_response, args=(y), result_ttl=0)
            except:
                mod.display("ASYNC_HTTP",
                            "ERROR",
                            "Could not enqueue job to Redis in func request_poller") 

# 需要导入模块: import rq [as 别名]
# 或者: from rq import Queue [as 别名]
def grade_project(project_id):
    project = Project.query.filter_by(id=project_id).first_or_404()
    with Connection(redis.from_url(current_app.config['REDIS_URL'])):
        q = Queue()
        task = q.enqueue(
            create_task,
            project.url,
            current_app.config["OPENFAAS_URL"]
        )
    response_object = {
        'status': 'success',
        'data': {
            'task_id': task.get_id()
        }
    }
    return jsonify(response_object), 202 

# 需要导入模块: import rq [as 别名]
# 或者: from rq import Queue [as 别名]
def run_task():
    words = request.form['words']
    q = Queue()
    task = q.enqueue(create_task, words)
    response_object = {
        'status': 'success',
        'data': {
            'task_id': task.get_id()
        }
    }
    return jsonify(response_object), 202