Python ThreadPool.spawn方法代码示例

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import spawn [as 别名]
class ThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor):
  """
  A version of :class:`concurrent.futures.ThreadPoolExecutor` that
  always uses native threads, even when threading is monkey-patched.

  TODO: Gevent has also implemented [ThreadPoolExecutor](https://github.com/gevent/gevent/blob/master/src/gevent/threadpool.py#L454)
   to be released in next version 1.2.0. We should move to that implementation when it is released.
  """

  def __init__(self, max_workers):
    super(ThreadPoolExecutor, self).__init__(max_workers)
    self._threadpool = ThreadPool(max_workers)

  def submit(self, fn, *args, **kwargs):
    future = super(ThreadPoolExecutor, self).submit(fn, *args, **kwargs)
    with self._shutdown_lock:
      work_item = self._work_queue.get()
      assert work_item.fn is fn

    self._threadpool.spawn(work_item.run)
    return future

  def shutdown(self, wait=True):
    super(ThreadPoolExecutor, self).shutdown(wait)
    self._threadpool.kill()

  kill = shutdown  # greentest compat

  def _adjust_thread_count(self):
    # Does nothing. We don't want to spawn any "threads",
    # let the threadpool handle that.
    pass

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import spawn [as 别名]
    def run_proxy_parse(self, url, proxy_type):
        try:
            for i in range(3):
                #这里对url进行重新组装
                url = url[0:-1] + str(i+1)
                resp = requests.get(url, timeout=5)
                response = etree.HTML(resp.content.decode('utf-8'))
                #解析代理IP，获取代理ip列表
                proxy_ip_list = []
                for item in response.xpath('//tr[contains(@style,"font-size")]'):
                    ip = item.xpath('./td[1]/text()')[0].strip()
                    port = item.xpath('./td[2]/text()')[0].strip()
                    proto = item.xpath('./td[4]/text()')[0].strip()
                    if proto.lower() == 'https':
                        continue
                    proxy_ip = proto.lower() + '://' + ip + ':' + port
                    if proxy_ip not in proxy_ip_list:
                        proxy_ip_list.append(proxy_ip)

                #对每个IP进行测试，留下可用的代理ip
                pool = ThreadPool(len(proxy_ip_list))
                for pip in proxy_ip_list:
                    pool.spawn(self.test_single_proxy, pip, self.test_urls[proxy_type.lower()])
                pool.join()
            
        except Exception, e:
            pass

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import spawn [as 别名]
 def spawn_dashboard(self, job, port):
     print 'Spawning dashboard...'
     sp_dashboard = create_fetch_dashboard(job)
     tpool = ThreadPool(2)
     tpool.spawn(sp_dashboard.serve,
                 use_reloader=False,
                 static_prefix='static',
                 port=port,
                 static_path=dashboard._STATIC_PATH)

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import spawn [as 别名]
    def admin_init(self):
        if self.coll_dir:
            self.load_coll_dir()

        if self.inputs:
            if self.load_cache():
                return

            pool = ThreadPool(maxsize=1)
            pool.spawn(self.safe_auto_load_warcs)

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import spawn [as 别名]
 def _get_messages(self):
     # Emulate batch messages by polling rabbitmq server multiple times
     pool = ThreadPool(settings.POLLER_CONFIG["batchsize"])
     for i in range(settings.POLLER_CONFIG["batchsize"]):
         if settings.QUEUE_TYPE in ["SQS", "sqs"]:
             pool.spawn(self._get_sqs_messages)
         elif settings.QUEUE_TYPE in ["RABBITMQ", "rabbitmq"]:
             pool.spawn(self._get_rabbitmq_messages, i)
         else:
             raise ValueError(
                 'Incorrect value "%s" for QUEUE_TYPE in %s' % (settings.QUEUE_TYPE, settings.SETTINGS_MODULE)
             )

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import spawn [as 别名]
 def vote(self):
     self._voted = False
     if self.sesskey is None or self.choice_id is None:
         self.get_info()
     print("[+] Start threadpool")
     pool = ThreadPool(10)
     for _ in range(11):
         pool.spawn(self._vote, _)
     gevent.wait()
     if self._voted:
         self.vote = True
         print("[+] Vote Success", time.ctime())

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import spawn [as 别名]
def fetch_controller(listname):
    port = find_port()
    if port:
        tpool = ThreadPool(2)
        tpool.spawn(start_fetch, listname, port)
        ret = {'port': port,
               'name': listname,
               'url': 'http://localhost:' + str(port),
               'status': 'running'}
    else:
        ret = {'port': port,
               'name': listname,
               'url': '',
               'status': 'occupied'}
    return ret

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import spawn [as 别名]
class Worker(object):
    def __init__(self, url):
        self.url = url
        self.headers = {'content-type': 'application/json'}
        self.id = 0
        self.post = functools.partial(requests.post, self.url, headers=self.headers)
        self.pool = ThreadPool(100)

    def _call(self, method, *args):
        payload = dict(method=method, params=args, jsonrpc="2.0", id=self.id)
        self.id += 1
        response = self.post(json.dumps(payload)).json()
        return response

    def _async_call(self, method, *args):
        payload = dict(method=method, params=args, jsonrpc="2.0", id=self.id)
        self.id += 1

        def _delayed_call(pl):
            return self.post(json.dumps(pl)).json()

        return Future(self.pool.spawn(_delayed_call, payload))

    def tell(self, method, *args):
        self._async_call(method, *args)

    def ask(self, method, *args):
        return self._async_call(method, *args)

    def join(self):
        self.pool.join()

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import spawn [as 别名]
 def _run(self):
     """Main loop; reap and process pkts"""
     try:
         args = self.orbname, self.select, self.reject
         print repr(self.orbname)
         print repr(args)
         with OrbreapThr(*args, timeout=1, queuesize=8, after=self.tafter) as orbreapthr:
             log.info("Connected to ORB %s %s %s" % (self.orbname, self.select,
                                                     self.reject))
             self.timeoff = self.timeon = datetime.utcnow()
             spawn(self._status_printer).link_exception(self._janitor)
             threadpool = ThreadPool(maxsize=1)
             try:
                 while True:
                     try:
                         success, value = threadpool.spawn(
                                 wrap_errors, (Exception,), orbreapthr.get, [], {}).get()
                         timestamp = datetime.utcnow()
                         if not success:
                             raise value
                     except (Timeout, NoData), e:
                         log.debug("orbreapthr.get exception %r" % type(e))
                         pass
                     else:
                         if value is None:
                             raise Exception('Nothing to publish')
                         self._process(value, timestamp)
             finally:
                 # This blocks until all threads in the pool return. That's
                 # critical; if the orbreapthr dies before the get thread,
                 # segfaults ensue.
                 threadpool.kill()
     except Exception, e:
         log.error("OrbPktSrc terminating due to exception", exc_info=True)
         raise

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import spawn [as 别名]
 def _run(self):
     try:
         args = self.orbname, self.select, self.reject
         # TODO Review this queue size
         # TODO Review reasoning behind using OrbreapThr vs. normal ORB API
         # I think it had something to do with orb.reap() blocking forever
         # on comms failures; maybe we could create our own orbreapthr
         # implementation?
         with OrbreapThr(*args, timeout=1, queuesize=orbreapthr_queuesize) as orbreapthr:
             log.info("Connected to ORB %s %s %s" % (self.orbname, self.select,
                                                     self.reject))
             threadpool = ThreadPool(maxsize=1)
             try:
                 while True:
                     try:
                         success, value = threadpool.spawn(
                                 wrap_errors, (Exception,), orbreapthr.get, [], {}).get()
                         timestamp = datetime.utcnow()
                         if not success:
                             raise value
                     except (Timeout, NoData), e:
                         log.debug("orbreapthr.get exception %r" % type(e))
                         pass
                     else:
                         if value is None:
                             raise Exception('Nothing to publish')
                         self._publish(value, timestamp)
             finally:
                 # This blocks until all threads in the pool return. That's
                 # critical; if the orbreapthr dies before the get thread,
                 # segfaults ensue.
                 threadpool.kill()
     except Exception, e:
         log.error("OrbPktSrc terminating due to exception", exc_info=True)
         raise

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import spawn [as 别名]
 def targets(self, activity):
     activities = self.get_contacts_by_activity[activity['id']]
     contacts = [int(c) for c in activities[TARGETS]]
     pool = ThreadPool(THREADS)
     contacts = [pool.spawn(self.get_contact, c) for c in contacts]
     gevent.wait()
     contacts = [c.get()['sort_name'] for c in contacts]
     return ', '.join(contacts)

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import spawn [as 别名]
    def test_auto_timeout_client__short_timeout_on_stuck_server(self):
        import time
        from threading import Event

        wait_for_start = Event()
        wait_for_close = Event()

        def thread_server(wait_for_start, wait_for_close):
            try:
                print(("starting server, hub: {}".format(gevent.hub.get_hub())))
                with logbook.NullHandler().applicationbound():
                    with server_context(FooService(), max_response_time=0.1):
                        print("server started.")
                        wait_for_start.set()
                        while not wait_for_close.is_set():
                            gevent.sleep(0.1)
            except:
                import traceback
                traceback.print_exc()

        from gevent.threadpool import ThreadPool

        t = ThreadPool(1)
        t.size = 1
        t.spawn(thread_server, wait_for_start, wait_for_close)

        try:
            print(("starting client, hub: {}".format(gevent.hub.get_hub())))
            client = AutoTimeoutClient(ZeroRPCClientTransport.create_tcp(8192), timeout_calc_func=lambda n: n * 2)
            wait_for_start.wait()
            print("client started.")
            t1 = time.time()
            self.assertRaises(TimeoutExpired, client.stuck_call)
            t2 = time.time()
            # This test should always pass although we're dealing with timing and non-deterministic measures since
            # stuck_call() is stuck for an entire second while we're comparing time to 0.2 (almost an order of a
            # magnitude)
            self.assertAlmostEqual(0.2, t2 - t1, delta=0.2)
        finally:
            wait_for_close.set()
            t.join()

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import spawn [as 别名]
def bench_spawn_wait(loops):
    pool = ThreadPool(1)

    t0 = perf.perf_counter()

    for _ in xrange(loops):
        for _ in xrange(N):
            r = pool.spawn(noop)
            r.get()

    pool.join()
    pool.kill()
    return perf.perf_counter() - t0

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import spawn [as 别名]
def my_task(self):
    min_pool = ThreadPool(2)

    self.update_state(state='PROGRESS',
                      meta={'current': 0, 'total': 2,
                            'status': 'PENDING'})

    # spawn all the threads
    greenlets = [min_pool.spawn(_consumer, i) for i in xrange(100)]
    # wait till the finish
    min_pool.join()

    # get the result
    response = [greenlet.get() for greenlet in greenlets]
    return {'current': 100, 'total': 100, 'status': 'Task completed!',
            'result': len(response)}

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import spawn [as 别名]
    def _get_related_contacts(self, relationships):
        """Return a list of related contacts from a CiviCRM server.
        Run the API calls concurrently to speed up the process.

        :param relationships: [required] relationship information of the
            selected contact
        :type relationships: list of dictionaries with relationship
            information
        :returns: dictionary with contact information dictionaries
        """
        contacts = []
        for r in relationships:
            if int(r["contact_id_a"]) == self.contact_id:
                contacts.append(r["contact_id_b"])
            elif int(r["contact_id_b"]) == self.contact_id:
                contacts.append(r["contact_id_a"])
        pool = ThreadPool(THREADS)
        contacts = [pool.spawn(self.get_contact, c) for c in contacts]
        gevent.wait()
        contacts = [c.get() for c in contacts]
        # make mapping easier by returning contact_id as dictionary key
        return dict([(c["contact_id"], c) for c in contacts])