Python queue.Queue方法代码示例

# 需要导入模块: import queue [as 别名]
# 或者: from queue import Queue [as 别名]
def main():
  print("Starting our Web Crawler")
  baseUrl = input("Website > ")
  numberOfThreads = input("No Threads > ")

  linksToCrawl = queue.Queue()
  urlLock = threading.Lock()
  linksToCrawl.put(baseUrl)
  haveVisited = []
  crawlers = []
  errorLinks = []

  for i in range(int(numberOfThreads)):
    crawler = Crawler(baseUrl, linksToCrawl, haveVisited, errorLinks, urlLock)
    crawler.start()
    crawlers.append(crawler)

  for crawler in crawlers:
    crawler.join()

  print("Total Number of Pages Visited {}".format(len(haveVisited)))
  print("Total Number of Pages with Errors {}".format(len(errorLinks))) 

# 需要导入模块: import queue [as 别名]
# 或者: from queue import Queue [as 别名]
def dd_iso_image(self, input_, output, gui_update, status_update):
        ''' Implementation for OS that use dd to write the iso image. 
        '''
        in_file_size = os.path.getsize(input_)
        cmd = [self.dd_exe, 'if=' + input_,
               'of=' + self.physical_disk(output), 'bs=1M']
        self.dd_iso_image_add_args(cmd, input_, output)
        kw_args = {
            'stdout' : subprocess.PIPE,
            'stderr' : subprocess.PIPE,
            'shell'  : False,
            }
        self.add_dd_iso_image_popen_args(kw_args)
        self.dd_iso_image_prepare(input, output, status_update)
        log('Executing => ' + str(cmd))
        dd_process = subprocess.Popen(cmd, **kw_args)
        output_q = queue.Queue()
        while dd_process.poll() is None:
            self.dd_iso_image_readoutput(dd_process, gui_update, in_file_size,
                                         output_q)
        output_lines = [output_q.get() for i in range(output_q.qsize())]
        for l in output_lines:
            log('dd: ' + l)
        return self.dd_iso_image_interpret_result(
            dd_process.returncode, output_lines) 

# 需要导入模块: import queue [as 别名]
# 或者: from queue import Queue [as 别名]
def main():
    nloops = randint(2, 20)
    q = Queue(32)

    threads = []
    for i in nfuncs:
        t = MyThread(funcs[i], (q, nloops), funcs[i].__name__)
        threads.append(t)

    for i in nfuncs:
        threads[i].start()

    for i in nfuncs:
        threads[i].join()

    print("all DONE") 

# 需要导入模块: import queue [as 别名]
# 或者: from queue import Queue [as 别名]
def __init__(self, batch_size, input_length, nthreads=6, web_viz=False):
        super(RLDataIter, self).__init__()
        self.batch_size = batch_size
        self.input_length = input_length
        self.env = [self.make_env() for _ in range(batch_size)]
        self.act_dim = self.env[0].action_space.n

        self.state_ = None

        self.reset()

        self.provide_data = [mx.io.DataDesc('data', self.state_.shape, np.uint8)]

        self.web_viz = web_viz
        if web_viz:
            self.queue = queue.Queue()
            self.thread = Thread(target=make_web, args=(self.queue,))
            self.thread.daemon = True
            self.thread.start()

        self.nthreads = nthreads
        if nthreads > 1:
            self.pool = multiprocessing.pool.ThreadPool(6) 

# 需要导入模块: import queue [as 别名]
# 或者: from queue import Queue [as 别名]
def itn_handler(host, port):  # type: (str, int) -> Iterator[Queue]
    """
    Usage::

        with itn_handler(ITN_HOST, ITN_PORT) as itn_queue:
            # ...complete PayFast payment...
            itn_data = itn_queue.get(timeout=2)
    """
    server_address = (host, port)
    http_server = HTTPServer(server_address, PayFastITNHandler)
    http_server.itn_queue = Queue()  # type: ignore

    executor = ThreadPoolExecutor(max_workers=1)
    executor.submit(http_server.serve_forever)
    try:
        yield http_server.itn_queue  # type: ignore
    finally:
        http_server.shutdown() 

# 需要导入模块: import queue [as 别名]
# 或者: from queue import Queue [as 别名]
def extractInfo(self):
        try:
            while not self.exit:
                try:
                    frame = self.frame_queue.get(block=True, timeout=1)
                except queue.Empty:
                    print("Queue empty")
                    continue
                try:
                    # Publish new image
                    msg = self.bridge.cv2_to_imgmsg(frame, 'rgb8')
                    if not self.exit:
                        self.image_publisher.publish(msg)
                except CvBridgeError as e:
                    print("Error Converting cv image: {}".format(e.message))
                self.frame_num += 1
        except Exception as e:
            print("Exception after loop: {}".format(e))
            raise 

# 需要导入模块: import queue [as 别名]
# 或者: from queue import Queue [as 别名]
def _check_and_execute(self):
        wakeup_queue = self._wakeup_queue
        while 1:
            (next_expired_time, expired_timers) = self._get_expired_timers()
            for timer in expired_timers:
                try:
                    # Note, please make timer callback effective/short
                    timer()
                except Exception:
                    logging.error(traceback.format_exc())

            self._reset_timers(expired_timers)

            sleep_time = _calc_sleep_time(next_expired_time)
            try:
                wakeup = wakeup_queue.get(timeout=sleep_time)
                if wakeup is TEARDOWN_SENTINEL:
                    break
            except Queue.Empty:
                pass
        logging.info('TimerQueue stopped.') 

# 需要导入模块: import queue [as 别名]
# 或者: from queue import Queue [as 别名]
def __init__(self, job_scheduler, event_writer):
        """
        @configs: a list like object containing a list of dict
        like object. Each element shall implement dict.get/[] like interfaces
        to get the value for a key.
        @job_scheduler: schedulering the jobs. shall implement get_ready_jobs
        @event_writer: write_events
        """

        self._settings = self._read_default_settings()
        self._settings["daemonize_thread"] = False
        self._event_writer = event_writer
        self._wakeup_queue = queue.Queue()
        self._scheduler = job_scheduler
        self._timer_queue = tq.TimerQueue()
        self._executor = ce.ConcurrentExecutor(self._settings)
        self._started = False 

# 需要导入模块: import queue [as 别名]
# 或者: from queue import Queue [as 别名]
def __init__(self, loader, split, ix_gen, max_items):
        """
        :param loader: Object responsible for loading samples.
        :param split: train, val or test.
        :param ix_gen: An infinite generator yielding tuples (ix, n).
        :param max_items: The maximum number of samples to have in the
                          queue.
        """
        super().__init__()
        self.ix_gen = ix_gen
        self.outq = queue.Queue(max_items)
        self.split = split
        self.loader = loader
        self.daemon = True 

# 需要导入模块: import queue [as 别名]
# 或者: from queue import Queue [as 别名]
def __get__(self, inst, owner):
        if inst is None:
            return self
        key = self._func.__get__(inst, type(inst))
        if key not in self._queues:
            self._queues[key] = queue.Queue()
            self._threads[key] = threading.Thread(target=self._thread_loop, args=[inst, key])
            self._threads[key].daemon = True
            self._threads[key].start()
        return lambda *a, **k: self._queues[key].put((a, k)) 

# 需要导入模块: import queue [as 别名]
# 或者: from queue import Queue [as 别名]
def iter_responses(urls, data=None, timeout=30):
    responses = queue.Queue()
    threads = []
    _req = lambda url, data, timeout: responses.put(http_request(url, data, timeout))
    for url in urls:
        threads.append(threading.Thread(target=_req, args=(url, data, timeout)))
        threads[-1].daemon = True
        threads[-1].start()
    for thread in threads:
        thread.join()
    responses.put(None)
    for response in iter(responses.get, None):
        yield response 

# 需要导入模块: import queue [as 别名]
# 或者: from queue import Queue [as 别名]
def mySubscriber(queue):
  while True:
    item = queue.get()
    if item is None:
      break
    print("{} removed {} from the queue".format(threading.current_thread(), item))
    print("Queue Size is now: {}".format(queue.qsize()))
    queue.task_done() 

# 需要导入模块: import queue [as 别名]
# 或者: from queue import Queue [as 别名]
def preload_streamed_response_content_async(requests_response_obj, buffer_queue):
    """
    stream模式下, 预读远程响应的content
    :param requests_response_obj:
    :type buffer_queue: queue.Queue
    """
    for particle_content in requests_response_obj.iter_content(stream_transfer_buffer_size):
        try:
            buffer_queue.put(particle_content, timeout=10)
        except queue.Full:  # coverage: exclude
            traceback.print_exc()
            exit()
        if verbose_level >= 3: dbgprint('BufferSize', buffer_queue.qsize())
    buffer_queue.put(None, timeout=10)
    exit() 

# 需要导入模块: import queue [as 别名]
# 或者: from queue import Queue [as 别名]
def __init__(
            self, queue, target
    ):
        """
        :type queue: queue.Queue
        """
        super(StoppableThread, self).__init__()
        self.task_func = target
        self.queue = queue
        self.daemon = True
        self._stopped = False 

# 需要导入模块: import queue [as 别名]
# 或者: from queue import Queue [as 别名]
def __init__(self):
        self.recv_queue = Queue()
        self.send_queue = Queue()