Python queue.Empty方法代码示例

# 需要导入模块: from gevent import queue [as 别名]
# 或者: from gevent.queue import Empty [as 别名]
def close(self):
        """Immediately close all connections and stop workers."""
        if not self.is_running:
            return

        self._state = CLOSED
        self.logger.debug('closing connection(s)')

        while True:
            try:
                conn = self._connections.get(block=False)
            except Empty:
                break

            conn.close_stream()

        self.on_close.send(self) 

# 需要导入模块: from gevent import queue [as 别名]
# 或者: from gevent.queue import Empty [as 别名]
def manage(self):
        """"
        Manage will hand out work when the appropriate Worker is free.
        The manager timeout must be less than worker timeout, or else, the
        workers will be idled and shutdown.
        """
        try:
            while True:
                for name, workgroup in self.workgroups.items():
                    for qname, q in self.qitems.items():
                        if name == qname: # workgroup name must match tracker name
                            # a tracker with the same name as workgroup name, is...
                            # ...effectively, the workgroup's task queue, so now...
                            # assign a task to a worker from the workgroup's task queue
                            for worker in workgroup:
                                one_task = q.get(timeout=self.mgr_qtimeout)
                                worker.tasks.put(one_task)
                gevent.sleep(0)
        except Empty:
            self.mgr_no_work = True
            if self.mgr_should_stop:
                logger.info(f"Assigned all {name} work. I've been told I should stop.")
                self.should_stop = True
            else:
                logger.info(f"Assigned all {name} work. Awaiting more tasks to assign.") 

# 需要导入模块: from gevent import queue [as 别名]
# 或者: from gevent.queue import Empty [as 别名]
def _concurrent_execute(self, context, start_req, parser, pool, pool_size):
        queue = Queue()  # 任务队列

        # 将初始化请求加入任务队列
        for r in start_req:
            queue.put_nowait(r)

        if pool is None:
            pool = GeventPool(pool_size)

        greenlets = []

        while True:
            try:
                req = self._check_req(queue.get(timeout=1))
                if req.parser is None:
                    req.parser = parser
                greenlets.append(pool.spawn(req, context, queue))
            except Empty:
                break

        return [greenlet.get() for greenlet in greenlets] 

# 需要导入模块: from gevent import queue [as 别名]
# 或者: from gevent.queue import Empty [as 别名]
def test_anonymous_no_concurrent_limit_because_remain_count(
        client, client_ip, mocker, url):
    mocker.patch.object(settings, 'CONCURRENT_LIMITER_SETTINGS', {
        '__anonymous__': {
            'ttl': 100,
            'capacity': 100,
        }
    })

    def worker(queue):
        response = client.get(url)
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    with raises(Empty):
        queue.get_nowait() 

# 需要导入模块: from gevent import queue [as 别名]
# 或者: from gevent.queue import Empty [as 别名]
def test_anonymous_no_rate_limit_because_remain_count(
        client, client_ip, mocker, url):
    mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', {
        '__anonymous__': {
            'rate': 100,
            'capacity': 300,
        }
    })

    def worker(queue):
        response = client.get(url)
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    with raises(Empty):
        queue.get_nowait() 

# 需要导入模块: from gevent import queue [as 别名]
# 或者: from gevent.queue import Empty [as 别名]
def test_logged_no_rate_limit_because_remain_count(
        client, client_ip, test_user, test_token, mocker):
    mocker.patch.object(settings, 'RATE_LIMITER_SETTINGS', {
        test_user.username: {
            'rate': 100,
            'capacity': 300,
        }
    })

    def worker(queue):
        response = client.get(
            '/api/need_login', headers={
                'Authorization': test_token,
            })
        if response.status_code == 429:
            queue.put(429)

    greenlets = []
    queue = Queue()
    for _ in range(3):
        greenlets.append(gevent.spawn(worker, queue))

    gevent.joinall(greenlets)
    with raises(Empty):
        queue.get_nowait() 

# 需要导入模块: from gevent import queue [as 别名]
# 或者: from gevent.queue import Empty [as 别名]
def update_users(self):
        already_updated = set()

        while True:
            # Only update so many at a time
            if len(already_updated) > 10000:
                return

            try:
                user_id, data = self.user_updates.get_nowait()
            except Empty:
                return

            if user_id in already_updated:
                continue

            already_updated.add(user_id)

            try:
                User.update(**data).where(User.user_id == user_id).execute()
            except:
                self.log.exception('Failed to update user %s: ', user_id) 

# 需要导入模块: from gevent import queue [as 别名]
# 或者: from gevent.queue import Empty [as 别名]
def _wait_for_ssh(queue, ssh, command, timeout=1, attempts=40):
    """Wait until a successful connection to the ssh endpoint can be made."""
    try:
        host, port = queue.get(timeout=timeout * attempts)
    except g_queue.Empty:
        cli.bad_exit('No SSH endpoint found.')

    for _ in six.moves.range(attempts):
        _LOGGER.debug('Checking SSH endpoint %s:%s', host, port)
        if _connect(host, port):
            run_ssh(host, port, ssh, list(command))
            break  # if run_ssh doesn't end with os.execvp()...

        try:
            host, port = queue.get(timeout=timeout)
            queue.task_done()
        except g_queue.Empty:
            pass

    # Either all the connection attempts failed or we're after run_ssh
    # (not resulting in os.execvp) so let's "clear the queue" so the thread
    # can join
    queue.task_done() 

# 需要导入模块: from gevent import queue [as 别名]
# 或者: from gevent.queue import Empty [as 别名]
def _get_connection(self, block=True, timeout=None):
        if not self.is_running:
            raise NSQException('producer not running')

        while True:
            try:
                conn = self._connections.get(block=block, timeout=timeout)
            except Empty:
                raise NSQNoConnections

            if conn.is_connected:
                return conn

            # Discard closed connections 

# 需要导入模块: from gevent import queue [as 别名]
# 或者: from gevent.queue import Empty [as 别名]
def spawn_spider(self, **kwargs):
        """
        Start and execute.
        """
        try:
            while True:
                task = self.tasks.get(timeout=self.qtimeout)  # decrements queue by 1
                logger.info(f'Worker {self.name}-{self.number} got task {task}')
                spider = self.get_spider(task, **kwargs)
                spider.start_http_session(**self.http_session)
                # OK, right here is where we wait for the spider to return a result.
                self.result(spider, task)
        except Empty:
            logger.info(f'Quitting time for worker {self.name}-{self.number}!') 

# 需要导入模块: from gevent import queue [as 别名]
# 或者: from gevent.queue import Empty [as 别名]
def await_data(connection, config):
    messages = []
    # block for first message - wake up after a while
    try:
        messages.extend(connection.queue.get(timeout=config["wake_connections_after"]))
    except Empty:
        pass
    # get more messages if enqueued takes up total 0.25
    while True:
        try:
            messages.extend(connection.queue.get(timeout=0.25))
        except Empty:
            break
    return messages 

# 需要导入模块: from gevent import queue [as 别名]
# 或者: from gevent.queue import Empty [as 别名]
def save_search_result_with_queue(queue):
    while 1:
        try:
            p = queue.get(timeout=0)
        except Empty:
            break

        save_search_result(p, queue)
    print 'stopping crawler...' 

# 需要导入模块: from gevent import queue [as 别名]
# 或者: from gevent.queue import Empty [as 别名]
def save_search_result_with_queue(queue):
    while 1:
        try:
            p = queue.get(timeout=0)
        except Empty:
            break

        print p, 'page'
        save_search_result(p, queue)
    print 'stopping crawler...' 

# 需要导入模块: from gevent import queue [as 别名]
# 或者: from gevent.queue import Empty [as 别名]
def _spawn_worker(self):
        def worker():
            while self._running:
                try:
                    func = self.hook_queue.get(timeout=1)
                    self._empty.clear()
                    try:
                        func()
                    except Exception:
                        logger.exception('Webhook task unexpected failed.')
                        capture_exception(data=None)
                except Empty:
                    self._empty.set()
                    continue
        return spawn(worker) 

# 需要导入模块: from gevent import queue [as 别名]
# 或者: from gevent.queue import Empty [as 别名]
def test_session_with_max_lifetime(
        zk, test_application_name, long_poll, mocker):
    mocker.patch.object(settings, 'LONG_POLLING_LIFE_SPAN_JITTER', 10)
    queue = long_poll(life_span=1)
    queue.get(timeout=5)
    gevent.sleep(1.1)

    zk.create(
        '/huskar/config/%s/alpha/DB_URL' % test_application_name, 'mysql://',
        makepath=True)

    with raises(Empty):
        queue.get(timeout=1.1)