Python threadpool.ThreadPool类代码示例

class ELExpression():
    def __init__(self):
        self.result = []
        self.pool = ThreadPool(10)
        self.q = []
        self.payload = '{1000-121}'
        self.match = '879'

    def putinqueue(self, info):
        try:

            url = info[0]
            data = info[1]
            current = data if data else url
            for k in re.finditer(r'\=(?P<value>.*?)(?:$|&)', current):
                value = k.group('value')
                payload = current.replace(value, self.payload)
                if data:
                    self.q.append((url, payload))
                else:
                    self.q.append((payload, data))
        except:
            traceback.print_exc()

    def Fuzz(self, info):
        try:
            url = info[0]
            data = info[1]
            if data:
                try:
                    r = requests.post(url, data=data, timeout=10, verify=False)
                    content = r.content
                except:
                    content = ''
            else:
                try:
                    print "Req ::" + url
                    r = requests.get(url, timeout=10, verify=False)
                    content = r.content
                except:
                    content = ''
                if self.match in content:
                    msg = 'find vulnerable url'
                    logging.info(msg)
                    self.result.append(info)
        except:
            traceback.print_exc()

    def Scan(self, info):
        try:
            if isinstance(info, tuple):
                self.putinqueue(info)
            else:
                with open(info) as f:
                    ud = json.loads(f.read())
                for i in ud:
                    self.putinqueue(i)
            self.pool.map(self.Fuzz, self.q)
        except:
            traceback.print_exc()

 def _run(self):
     try:
         args = self.orbname, self.select, self.reject
         # TODO Review this queue size
         # TODO Review reasoning behind using OrbreapThr vs. normal ORB API
         # I think it had something to do with orb.reap() blocking forever
         # on comms failures; maybe we could create our own orbreapthr
         # implementation?
         with OrbreapThr(*args, timeout=1, queuesize=orbreapthr_queuesize) as orbreapthr:
             log.info("Connected to ORB %s %s %s" % (self.orbname, self.select,
                                                     self.reject))
             threadpool = ThreadPool(maxsize=1)
             try:
                 while True:
                     try:
                         success, value = threadpool.spawn(
                                 wrap_errors, (Exception,), orbreapthr.get, [], {}).get()
                         timestamp = datetime.utcnow()
                         if not success:
                             raise value
                     except (Timeout, NoData), e:
                         log.debug("orbreapthr.get exception %r" % type(e))
                         pass
                     else:
                         if value is None:
                             raise Exception('Nothing to publish')
                         self._publish(value, timestamp)
             finally:
                 # This blocks until all threads in the pool return. That's
                 # critical; if the orbreapthr dies before the get thread,
                 # segfaults ensue.
                 threadpool.kill()
     except Exception, e:
         log.error("OrbPktSrc terminating due to exception", exc_info=True)
         raise

class ThreadPoolExecutor(concurrent.futures.ThreadPoolExecutor):
  """
  A version of :class:`concurrent.futures.ThreadPoolExecutor` that
  always uses native threads, even when threading is monkey-patched.

  TODO: Gevent has also implemented [ThreadPoolExecutor](https://github.com/gevent/gevent/blob/master/src/gevent/threadpool.py#L454)
   to be released in next version 1.2.0. We should move to that implementation when it is released.
  """

  def __init__(self, max_workers):
    super(ThreadPoolExecutor, self).__init__(max_workers)
    self._threadpool = ThreadPool(max_workers)

  def submit(self, fn, *args, **kwargs):
    future = super(ThreadPoolExecutor, self).submit(fn, *args, **kwargs)
    with self._shutdown_lock:
      work_item = self._work_queue.get()
      assert work_item.fn is fn

    self._threadpool.spawn(work_item.run)
    return future

  def shutdown(self, wait=True):
    super(ThreadPoolExecutor, self).shutdown(wait)
    self._threadpool.kill()

  kill = shutdown  # greentest compat

  def _adjust_thread_count(self):
    # Does nothing. We don't want to spawn any "threads",
    # let the threadpool handle that.
    pass

    def run_proxy_parse(self, url, proxy_type):
        try:
            for i in range(3):
                #这里对url进行重新组装
                url = url[0:-1] + str(i+1)
                resp = requests.get(url, timeout=5)
                response = etree.HTML(resp.content.decode('utf-8'))
                #解析代理IP，获取代理ip列表
                proxy_ip_list = []
                for item in response.xpath('//tr[contains(@style,"font-size")]'):
                    ip = item.xpath('./td[1]/text()')[0].strip()
                    port = item.xpath('./td[2]/text()')[0].strip()
                    proto = item.xpath('./td[4]/text()')[0].strip()
                    if proto.lower() == 'https':
                        continue
                    proxy_ip = proto.lower() + '://' + ip + ':' + port
                    if proxy_ip not in proxy_ip_list:
                        proxy_ip_list.append(proxy_ip)

                #对每个IP进行测试，留下可用的代理ip
                pool = ThreadPool(len(proxy_ip_list))
                for pip in proxy_ip_list:
                    pool.spawn(self.test_single_proxy, pip, self.test_urls[proxy_type.lower()])
                pool.join()
            
        except Exception, e:
            pass

class Worker(object):
    def __init__(self, url):
        self.url = url
        self.headers = {'content-type': 'application/json'}
        self.id = 0
        self.post = functools.partial(requests.post, self.url, headers=self.headers)
        self.pool = ThreadPool(100)

    def _call(self, method, *args):
        payload = dict(method=method, params=args, jsonrpc="2.0", id=self.id)
        self.id += 1
        response = self.post(json.dumps(payload)).json()
        return response

    def _async_call(self, method, *args):
        payload = dict(method=method, params=args, jsonrpc="2.0", id=self.id)
        self.id += 1

        def _delayed_call(pl):
            return self.post(json.dumps(pl)).json()

        return Future(self.pool.spawn(_delayed_call, payload))

    def tell(self, method, *args):
        self._async_call(method, *args)

    def ask(self, method, *args):
        return self._async_call(method, *args)

    def join(self):
        self.pool.join()

 def _run(self):
     """Main loop; reap and process pkts"""
     try:
         args = self.orbname, self.select, self.reject
         print repr(self.orbname)
         print repr(args)
         with OrbreapThr(*args, timeout=1, queuesize=8, after=self.tafter) as orbreapthr:
             log.info("Connected to ORB %s %s %s" % (self.orbname, self.select,
                                                     self.reject))
             self.timeoff = self.timeon = datetime.utcnow()
             spawn(self._status_printer).link_exception(self._janitor)
             threadpool = ThreadPool(maxsize=1)
             try:
                 while True:
                     try:
                         success, value = threadpool.spawn(
                                 wrap_errors, (Exception,), orbreapthr.get, [], {}).get()
                         timestamp = datetime.utcnow()
                         if not success:
                             raise value
                     except (Timeout, NoData), e:
                         log.debug("orbreapthr.get exception %r" % type(e))
                         pass
                     else:
                         if value is None:
                             raise Exception('Nothing to publish')
                         self._process(value, timestamp)
             finally:
                 # This blocks until all threads in the pool return. That's
                 # critical; if the orbreapthr dies before the get thread,
                 # segfaults ensue.
                 threadpool.kill()
     except Exception, e:
         log.error("OrbPktSrc terminating due to exception", exc_info=True)
         raise

 def targets(self, activity):
     activities = self.get_contacts_by_activity[activity['id']]
     contacts = [int(c) for c in activities[TARGETS]]
     pool = ThreadPool(THREADS)
     contacts = [pool.spawn(self.get_contact, c) for c in contacts]
     gevent.wait()
     contacts = [c.get()['sort_name'] for c in contacts]
     return ', '.join(contacts)

def easy_parallelize_gevent(f, sequence):
    if not "gevent_pool" in PARALLEL_STRUCTURES:
        from gevent.threadpool import ThreadPool
        pool = ThreadPool(30000)
        PARALLEL_STRUCTURES["gevent_pool"] = pool
    pool = PARALLEL_STRUCTURES["gevent_pool"]
    result = pool.map(f, sequence)
    return result

 def spawn_dashboard(self, job, port):
     print 'Spawning dashboard...'
     sp_dashboard = create_fetch_dashboard(job)
     tpool = ThreadPool(2)
     tpool.spawn(sp_dashboard.serve,
                 use_reloader=False,
                 static_prefix='static',
                 port=port,
                 static_path=dashboard._STATIC_PATH)

    def admin_init(self):
        if self.coll_dir:
            self.load_coll_dir()

        if self.inputs:
            if self.load_cache():
                return

            pool = ThreadPool(maxsize=1)
            pool.spawn(self.safe_auto_load_warcs)

class TestJoinEmpty(TestCase):
    switch_expected = False

    def test(self):
        self.pool = ThreadPool(1)
        self.pool.join()

    def cleanup(self):
        # needed here because currently Greenlet.kill() switches out even if greenlet not started yet
        # XXX fix Greenlet.kill
        pass

 def _get_messages(self):
     # Emulate batch messages by polling rabbitmq server multiple times
     pool = ThreadPool(settings.POLLER_CONFIG["batchsize"])
     for i in range(settings.POLLER_CONFIG["batchsize"]):
         if settings.QUEUE_TYPE in ["SQS", "sqs"]:
             pool.spawn(self._get_sqs_messages)
         elif settings.QUEUE_TYPE in ["RABBITMQ", "rabbitmq"]:
             pool.spawn(self._get_rabbitmq_messages, i)
         else:
             raise ValueError(
                 'Incorrect value "%s" for QUEUE_TYPE in %s' % (settings.QUEUE_TYPE, settings.SETTINGS_MODULE)
             )

 def vote(self):
     self._voted = False
     if self.sesskey is None or self.choice_id is None:
         self.get_info()
     print("[+] Start threadpool")
     pool = ThreadPool(10)
     for _ in range(11):
         pool.spawn(self._vote, _)
     gevent.wait()
     if self._voted:
         self.vote = True
         print("[+] Vote Success", time.ctime())

def fetch_controller(listname):
    port = find_port()
    if port:
        tpool = ThreadPool(2)
        tpool.spawn(start_fetch, listname, port)
        ret = {'port': port,
               'name': listname,
               'url': 'http://localhost:' + str(port),
               'status': 'running'}
    else:
        ret = {'port': port,
               'name': listname,
               'url': '',
               'status': 'occupied'}
    return ret

 def __init__(self):
     self.pool = ThreadPool(10)
     self.result = []
     self.port = "5984"
     self.q = []
     self.randomstrs = ['a', 'k', 'b', 'v', 'd', 'f', 'e', 'g']
     self.path = '_utils/index.html'