Python JoinableQueue.put_nowait方法代碼示例

# 需要導入模塊: from gevent.queue import JoinableQueue [as 別名]
# 或者: from gevent.queue.JoinableQueue import put_nowait [as 別名]
def recursive_crawl(url):
    all_urls = set()
    processed_urls = set()
    task_queue = JoinableQueue()

    def add_to_all(url):
        if url not in all_urls:
            print("Record url {}".format(url))
            all_urls.add(url)

    task_queue.put_nowait(url)

    # Start workers
    workers = []
    for i in xrange(10):
        workers.append(gevent.spawn(url_worker, i, processed_urls, add_to_all, task_queue))
    print("workers", len(workers))

    task_queue.join()

    print("Processed", len(processed_urls), "All", len(all_urls))
    print("Total latency", demo_helpers.TOTAL_LATENCY)

# 需要導入模塊: from gevent.queue import JoinableQueue [as 別名]
# 或者: from gevent.queue.JoinableQueue import put_nowait [as 別名]
            f.write(s.read())

        WORDS_STATUS['downloaded'].add(name)


if __name__ == '__main__':
    words = sys.argv[1:]

    queue_start = JoinableQueue()
    queue_to_search = JoinableQueue()
    queue_to_download = JoinableQueue()

    pbar = ProgressBar(maxval=len(words) * 3).start()

    for word in words:
        queue_start.put_nowait(word)

        CheckWord(queue_start, queue_to_search, pbar).start()
        SearchWord(queue_to_search, queue_to_download, pbar).start()
        Downloading(queue_to_download, pbar).start()

    queue_start.join()
    queue_to_search.join()
    queue_to_download.join()

    pbar.finish()

    exists = ', '.join(WORDS_STATUS['exists'])
    not_found = ', '.join(WORDS_STATUS['not_found'])
    downloaded = ', '.join(WORDS_STATUS['downloaded'])

# 需要導入模塊: from gevent.queue import JoinableQueue [as 別名]
# 或者: from gevent.queue.JoinableQueue import put_nowait [as 別名]

#.........這裏部分代碼省略.........
            fitness_cutoff = payload["fitness_cutoff"]
            self.pop.next_generation(current_gen, fitness_cutoff)

            self.evt.set()
            self.evt = None
        else:
            self._bad_auth(payload, body, start_response)


    def pop_enum (self, *args, **kwargs):
        """enumerate the Individuals in this shard of the Population"""
        payload = args[0]
        body = args[1]
        start_response = args[2]

        if (self.prefix == payload["prefix"]) and (self.shard_id == payload["shard_id"]):
            fitness_cutoff = payload["fitness_cutoff"]

            start_response('200 OK', [('Content-Type', 'application/json')])
            body.put(dumps(self.pop.enum(fitness_cutoff)))
            body.put("\r\n")
            body.put(StopIteration)
        else:
            self._bad_auth(payload, body, start_response)


    def pop_reify (self, *args, **kwargs):
        """test/add a newly generated Individual into the Population (birth)"""
        payload = args[0]
        body = args[1]
        start_response = args[2]

        if (self.prefix == payload["prefix"]) and (self.shard_id == payload["shard_id"]):
            self.reify_queue.put_nowait(payload)

            start_response('200 OK', [('Content-Type', 'text/plain')])
            body.put("Bokay\r\n")
            body.put(StopIteration)
        else:
            self._bad_auth(payload, body, start_response)


    def _response_handler (self, env, start_response):
        """handle HTTP request/response"""
        uri_path = env['PATH_INFO']
        body = Queue()

        ## NB: these handler cases can be collapsed into a common pattern
        ## except for config/stop -- later

        ##########################################
        # shard lifecycle endpoints

        if uri_path == '/shard/config':
            # configure the service to run a shard
            payload = loads(env['wsgi.input'].read())
            gl = Greenlet(self.shard_config, payload, body, start_response)
            gl.start()

        elif uri_path == '/shard/persist':
            # checkpoint the service state to durable storage
            payload = loads(env['wsgi.input'].read())
            print "POST", payload
            ## TODO
            start_response('200 OK', [('Content-Type', 'text/plain')])
            body.put("Bokay\r\n")

# 需要導入模塊: from gevent.queue import JoinableQueue [as 別名]
# 或者: from gevent.queue.JoinableQueue import put_nowait [as 別名]

#.........這裏部分代碼省略.........
    ######################################################################
    ## barrier pattern methods

    @contextmanager
    def wrap_task_event (self):
        """initialize a gevent.Event, to which the UnitOfWork will wait as a listener"""
        self._task_event = Event()
        yield

        # complete the Event, notifying the UnitOfWork which waited
        self._task_event.set()
        self._task_event = None


    def _consume_task_queue (self):
        """consume/serve requests until the task_queue empties"""
        while True:
            payload = self._task_queue.get()

            try:
                self._uow.perform_task(payload)
            finally:
                self._task_queue.task_done()


    def prep_task_queue (self):
        """prepare task_queue for another set of distributed tasks"""
        self._task_queue = JoinableQueue()
        spawn(self._consume_task_queue)


    def put_task_queue (self, payload):
        """put the given task definition into the task_queue"""
        self._task_queue.put_nowait(payload)


    def queue_wait (self, *args, **kwargs):
        """wait until all shards finished sending task_queue requests"""
        payload, start_response, body = self.get_response_context(args)

        if self.auth_request(payload, start_response, body):
            if self._task_event:
                self._task_event.wait()

            # HTTP response first, then initiate long-running task
            start_response('200 OK', [('Content-Type', 'text/plain')])
            body.put("Bokay\r\n")
            body.put(StopIteration)


    def queue_join (self, *args, **kwargs):
        """join on the task_queue, as a barrier to wait until it empties"""
        payload, start_response, body = self.get_response_context(args)

        if self.auth_request(payload, start_response, body):
            start_response('200 OK', [('Content-Type', 'text/plain')])
            body.put("join queue...\r\n")

            ## NB: TODO this step of emptying out the task_queue on
            ## shards could take a while on a large run... perhaps use
            ## a long-polling HTTP request or websocket instead?
            self._task_queue.join()

            body.put("done\r\n")
            body.put(StopIteration)