Python pool.Pool方法代码示例

# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Pool [as 别名]
def coroutine_response(self, size=None, stream=False, exception_handler=None, status_only=False):
        g_requests = list(self.__grequests)

        pool = Pool(size) if size else None
        jobs = [send(r, pool, stream=stream) for r in g_requests]
        gevent.joinall(jobs)

        ret = []
        if status_only:
            for request in g_requests:
                if request.response:
                    ret.append(copy.copy(True))
                else:
                    ret.append(copy.copy(False))
            return ret

        for request in g_requests:
            if request.response:
                ret.append(request.response)
            elif exception_handler:
                exception_handler(request, request.exception)



        return ret 

# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Pool [as 别名]
def __init__(self, client):
        """CDNClient allows loading and reading of manifests for Steam apps are used
        to list and download content

        :param client: logged in SteamClient instance
        :type  client: :class:`.SteamClient`
        """
        self.gpool = GPool(8)            #: task pool
        self.steam = client              #: SteamClient instance
        if self.steam:
            self.cell_id = self.steam.cell_id

        self.web = make_requests_session()
        self.depot_keys = {}             #: depot decryption keys
        self.manifests = {}              #: CDNDepotManifest instances
        self.app_depots = {}             #: app depot info
        self.beta_passwords = {}         #: beta branch decryption keys
        self.licensed_app_ids = set()    #: app_ids that the SteamClient instance has access to
        self.licensed_depot_ids = set()  #: depot_ids that the SteamClient instance has access to

        if not self.servers:
            self.fetch_content_servers()

        self.load_licenses() 

# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Pool [as 别名]
def set_spawn(self, spawn):
        if spawn == 'default':
            self.pool = None
            self._spawn = self._spawn
        elif hasattr(spawn, 'spawn'):
            self.pool = spawn
            self._spawn = spawn.spawn
        elif isinstance(spawn, integer_types):
            from gevent.pool import Pool
            self.pool = Pool(spawn)
            self._spawn = self.pool.spawn
        else:
            self.pool = None
            self._spawn = spawn
        if hasattr(self.pool, 'full'):
            self.full = self.pool.full
        if self.pool is not None:
            self.pool._semaphore.rawlink(self._start_accepting_if_started) 

# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Pool [as 别名]
def run(self):
        import gevent
        from gevent import monkey
        monkey.patch_all()
        from gevent import pool
        # default 200
        # g_pool = pool.Pool(200)
        g_pool = pool.Pool(self.coroutine)
        tasks = [g_pool.spawn(self.gen_traffic, url) for url in self.url_list]
        gevent.joinall(tasks)
        traffic_list = []
        for i in tasks:
            if i.value is not None:
                traffic_list.append(i.value)
        # save traffic for rescan
        Engine.save_traffic(traffic_list, self.id) 

# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Pool [as 别名]
def verify_async(case_list,coroutine):
        """
        Verify used gevent lib
        :param case_list:
        :param coroutine:
        :return:
        """
        from gevent import monkey
        monkey.patch_all()
        result = []
        geventPool = pool.Pool(coroutine)
        tasks = [geventPool.spawn(Verify.request_and_verify, case) for case in case_list]
        gevent.joinall(tasks)
        for i in tasks:
            if i.value is not None:
                result.append(i.value)
        print_info('Total Verify-Case is: %s, %s error happened.' % (len(case_list), Verify.ERROR_COUNT))
        return result 

# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Pool [as 别名]
def deduplicate(self, url_list):
        print 'Start to deduplicate for all urls.'
        filtered_path = self.file + '.filtered'
        if os.path.exists(filtered_path):
            print '%s has been filtered as %s.' % (self.file, filtered_path)
            with open(filtered_path)as f:
                filtered = f.read().split('\n')
                return filtered
        filtered = []
        # result = map(filter, url_list)
        from multiprocessing import cpu_count
        from multiprocessing.pool import Pool
        p=Pool(cpu_count())
        result=p.map(url_filter,url_list)
        for i in result:
            if isinstance(i, str):
                filtered.append(i)
        with open(filtered_path, 'w') as f:
            f.write('\n'.join(filtered))
        print 'Saved filtered urls to %s.' % filtered_path
        return filtered 

# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Pool [as 别名]
def web():
    from gevent.subprocess import run
    from gevent.pool import Pool

    def api():
        run('bin/run-web', shell=True)

    def webpack():
        run('command -v yarn && yarn run dev || npm run dev', shell=True)

    try:
        pool = Pool()
        pool.spawn(api)
        pool.spawn(webpack)
        pool.join()
    except KeyboardInterrupt:
        time.sleep(1) 

# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Pool [as 别名]
def multiappend(con, box, msgs, *, batch=None, threads=10):
    if not msgs:
        return

    if batch and len(msgs) > batch:
        def multiappend_inner(num, few):
            with con.new() as c:
                res = multiappend(c, box, few)
                log.debug('#%s multiappend %s messages', num, len(few))
                return res

        pool = Pool(threads)
        jobs = [
            pool.spawn(multiappend_inner, num, msgs[i:i+batch])
            for num, i in enumerate(range(0, len(msgs), batch))
        ]
        pool.join(raise_error=True)
        return ','.join(j.value for j in jobs)

    with con.lock:
        return _multiappend(con, box, msgs) 

# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Pool [as 别名]
def call_async(self, fn, *args):
        if not self.batches:
            return self.call(fn, *args)

        def get_exceptions():
            return [j.exception for j in jobs if j.exception]

        jobs = []
        pool = Pool(self.threads)
        for f in self._call(fn, *args):
            if pool.wait_available():
                if get_exceptions():
                    break
                jobs.append(pool.spawn(f))
        pool.join()

        exceptions = get_exceptions()
        if exceptions:
            raise ValueError('Exception in the pool: %s' % exceptions)
        return (f.value for f in jobs) 

# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Pool [as 别名]
def run(self, id_filename):
        s3 = S3Boto3Storage().connection
        obj = s3.Object(self.output_bucket, id_filename)
        arxiv_id_str = obj.get()["Body"].read().decode("utf-8")
        arxiv_ids = [s.strip() for s in arxiv_id_str.split() if s.strip()]

        # We can't access database inside our gevent pool because of max
        # connections, so first figure out which IDs we actually want to
        # render.
        arxiv_ids, source_paths = self.filter_unrenderable_ids(arxiv_ids)

        # Stagger the starting of jobs a bit so we don't break Hyper.sh
        def slow_arxiv_ids():
            for arxiv_id in arxiv_ids:
                yield arxiv_id
                time.sleep(0.1)

        pool = Pool(self.concurrency)
        manifest = pool.imap_unordered(self.render, slow_arxiv_ids(), source_paths)
        # Failed renders are None
        manifest = (obj for obj in manifest if obj)
        # Read the iterator, starting the actual processing
        manifest = list(manifest)
        self.write_manifest(manifest) 

# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Pool [as 别名]
def load(self, table):
        cache = {}

        def save(row):
            (query, values) = self.get_insert(row, table)
            try:
                prepared = cache[query]
            except:
                prepared = self.session.prepare(query)
                cache[query] = prepared
            bound = prepared.bind(values)
            self.session.execute(bound)

        pool = Pool(100)
        i = 0
        print "Loading {}".format(table)
        with ProgressBar(max_value=len(self.dataframe)) as p:
            for _ in pool.imap_unordered(save, self.iter()):
                i += 1
                if i % 10 == 0:
                    p.update(i) 

# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Pool [as 别名]
def _concurrent_execute(self, context, start_req, parser, pool, pool_size):
        queue = Queue()  # 任务队列

        # 将初始化请求加入任务队列
        for r in start_req:
            queue.put_nowait(r)

        if pool is None:
            pool = GeventPool(pool_size)

        greenlets = []

        while True:
            try:
                req = self._check_req(queue.get(timeout=1))
                if req.parser is None:
                    req.parser = parser
                greenlets.append(pool.spawn(req, context, queue))
            except Empty:
                break

        return [greenlet.get() for greenlet in greenlets] 

# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Pool [as 别名]
def _load_dns_servers(self):
        print '[+] Validate DNS servers ...'
        self.dns_servers = []
        pool = Pool(30)
        for server in open('dict/dns_servers.txt').xreadlines():
            server = server.strip()
            if server:
                pool.apply_async(self._test_server, (server,))
        pool.join()

        self.dns_count = len(self.dns_servers)
        sys.stdout.write('\n')
        print '[+] Found %s available DNS Servers in total' % self.dns_count
        if self.dns_count == 0:
            print '[ERROR] No DNS Servers available.'
            sys.exit(-1) 

# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Pool [as 别名]
def register(self):
        f = open('fail.txt', 'w')
        f.close()
        f = open('success.txt', 'w')
        f.close()
        p = Pool(200)
        f = open('邮箱.txt', 'w+')
        # emails = f.read().strip().split('\n')
        emails = list(map(lambda x: x.split('----')[0], f.read().strip().split('\n')))
        if emails and emails[0] == "":
            return
        # emails = ['nkzhuanyong' + str(i) + '@sina.com' for i in range(200)]
        passw = 'Aa123456'
        for i in emails:
            p.apply_async(self.reg, args=(i, passw))
        p.join()
        self.signalStatus.emit('完成') 

# 需要导入模块: from gevent import pool [as 别名]
# 或者: from gevent.pool import Pool [as 别名]
def getRootDomains(self):

		url = "http://www.iana.org/domains/root/db"
		soup = self.soup(url)
		link = soup.find('table', {'id': 'tld-table'})
		tlds = [anchor.text for anchor in link.find_all('a')]
		
		#Get RootDomains
		
		jobs = []
		links = []
		p = pool.Pool(20)

		for allRootDomains in tlds:
			https = "https://www."+self.org+allRootDomains
			http = "http://www."+self.org+allRootDomains
			#Check http(s) urls 
			l = [https,http]
			for url in l:
				jobs.append(p.spawn(self.getUrl, url))
		gevent.joinall(jobs)