Python gevent.pool方法代码示例

# 需要导入模块: import gevent [as 别名]
# 或者: from gevent import pool [as 别名]
def coroutine_response(self, size=None, stream=False, exception_handler=None, status_only=False):
        g_requests = list(self.__grequests)

        pool = Pool(size) if size else None
        jobs = [send(r, pool, stream=stream) for r in g_requests]
        gevent.joinall(jobs)

        ret = []
        if status_only:
            for request in g_requests:
                if request.response:
                    ret.append(copy.copy(True))
                else:
                    ret.append(copy.copy(False))
            return ret

        for request in g_requests:
            if request.response:
                ret.append(request.response)
            elif exception_handler:
                exception_handler(request, request.exception)



        return ret 

# 需要导入模块: import gevent [as 别名]
# 或者: from gevent import pool [as 别名]
def run(self):
		# 先判断结果是否存在过
		# 以及报告目录是否存在
		findreport(self.document)

		# 先检测一个不存在的地址
		# 如果能解析，则证明做了一些保护措施
		# 将对方地址记录下来，作为block的标准
		block_check_results = self.checkdomain('d312379196bd822558ca7dfb3c95ba61.'+self.options['target'],'block')
		
		if block_check_results:
			self.blockip = block_check_results[0]

		# 构建字典
		dic_list = (dic.strip('\n')+'.'+self.options['target'] for dic in open(getpath() + '/' +self.options['dictname'],'r'))
		# 协程爆破测试
		self.pool.map(self.checkdomain,dic_list) 

# 需要导入模块: import gevent [as 别名]
# 或者: from gevent import pool [as 别名]
def spawn_list(self):
        """"
        The spawn() method begins a new greenlet with the given arguments
        (which are passed to the greenlet constructor) and adds it to the
        collection of greenlets this group is monitoring.

        We return a list of the newly started greenlets, used in a later
        'joinall` call.

        :return: A list of the newly started greenlets.
        """

        # here, workgroups is a list of Type[BaseGroup] objects
        workgroups = [val for val in self.workgroups.values()]
        spawn_list = [self.pool.spawn(self.monitor, worker) for work_group in
                      workgroups for worker in work_group]

        # we get a blocking error if we spawn the manager first, so spawn it last
        spawn_list.append(self.pool.spawn(self.manage))

        return spawn_list 

# 需要导入模块: import gevent [as 别名]
# 或者: from gevent import pool [as 别名]
def main(version="1.0.0"):
    conf = Configuration(description="", version=version)
    conf.register_str("log_level", default="INFO",
                      help="The level of the log, such as debug, info, etc.")
    conf.register_str("log_file", default="", help="The file path of the log.")
    conf.register_int("thread_num", default=0, help="The size of the coroutine pool.")
    conf.parse()

    if conf.thread_num > 0:
        global taskpool, spawn
        taskpool = gevent.pool.Pool(size=conf.thread_num)
        spawn = taskpool.spawn

    init_logging(LOG, conf.log_level, conf.log_file)

    # TODO:) 

# 需要导入模块: import gevent [as 别名]
# 或者: from gevent import pool [as 别名]
def execute(self, context, start_req, parser=_default_parser,
                pool=None, pool_size=None):
        """
        :param context: 上下文对象
        :param start_req: 起始请求列表
        :param parser: Response对象解析器
        :param concurrent: 是否采用并发方式抓取
        :param pool: 指定已有的gevent pool
        """

        if pool or pool_size:
            # 并发请求
            return self._concurrent_execute(
                context, start_req, parser,
                pool, pool_size)
        else:
            # 同步请求
            return self._sync_execute(context, start_req, parser) 

# 需要导入模块: import gevent [as 别名]
# 或者: from gevent import pool [as 别名]
def _concurrent_execute(self, context, start_req, parser, pool, pool_size):
        queue = Queue()  # 任务队列

        # 将初始化请求加入任务队列
        for r in start_req:
            queue.put_nowait(r)

        if pool is None:
            pool = GeventPool(pool_size)

        greenlets = []

        while True:
            try:
                req = self._check_req(queue.get(timeout=1))
                if req.parser is None:
                    req.parser = parser
                greenlets.append(pool.spawn(req, context, queue))
            except Empty:
                break

        return [greenlet.get() for greenlet in greenlets] 

# 需要导入模块: import gevent [as 别名]
# 或者: from gevent import pool [as 别名]
def add(self, greenlet):
        source = greenlet.args[2][1][0] + ':' + str(greenlet.args[2][1][1])
        socket = greenlet.args[2][0]

        # With 1, we avoid the wait caused by the semaphore
        if self.free_count() < 2:
            # /!\ pool full, untracking oldest greenlet /!\
            oldest_source = self.open_connection[0]
            oldest_greenlet = self.open_connection_dico_ip[oldest_source]

            #kill the greenlet, this also close its associated socket
            self.killone(oldest_greenlet, block=False)

        #Add the connection to the dicos
        self.open_connection.append(source)
        self.open_connection_dico_ip[source] = greenlet
        self.open_connection_dico_green[str(greenlet)] = source
        gevent.pool.Pool.add( self, greenlet)

    # discard the greenlet, free one slot of the pool 

# 需要导入模块: import gevent [as 别名]
# 或者: from gevent import pool [as 别名]
def record_proxy_state(self, proxy, state):
        if not proxy: return
        proxy_url = proxy['http']
        if state == self.FAILED:
            if proxy_url in self._proxy_failed:
                self._proxy_failed[proxy_url] += 1
                if self._proxy_failed[proxy_url] > self._proxy_failed_threshold:
                    try:
                        print '!!! remove proxy: %s , left: [%s] !!!' % (
                            proxy_url,
                            len(self._pool)
                        )
                        # TODO proxy pool changed, changes should also be made to recording data structures such as _host_proxy_time
                        self._pool.remove(proxy)
                    except:
                        pass
            else:
                self._proxy_failed[proxy_url] = 1
        elif state == self.SUCCESS:
            if proxy_url in self._proxy_failed:
                if self._proxy_failed[proxy_url] > 0:
                    self._proxy_failed[proxy_url] -= 1
        else:
            print '!!!!! invalid proxy state: %s !!!!' % state 

# 需要导入模块: import gevent [as 别名]
# 或者: from gevent import pool [as 别名]
def _load_dns_servers(self):
        print '[+] Validate DNS servers ...'
        self.dns_servers = []
        pool = Pool(30)
        for server in open('dict/dns_servers.txt').xreadlines():
            server = server.strip()
            if server:
                pool.apply_async(self._test_server, (server,))
        pool.join()

        self.dns_count = len(self.dns_servers)
        sys.stdout.write('\n')
        print '[+] Found %s available DNS Servers in total' % self.dns_count
        if self.dns_count == 0:
            print '[ERROR] No DNS Servers available.'
            sys.exit(-1) 

# 需要导入模块: import gevent [as 别名]
# 或者: from gevent import pool [as 别名]
def task(pool_number, types=types, tags_dict=tags_dict, sorts=sorts):
    video_douban_ids = set(get_video_douban_ids())
    global video_douban_ids

    pool = Pool(pool_number)

    for type in types:
        for tag in tags_dict[type]:
            for sort in sorts:
                pool.spawn(
                    create_requests_and_save_datas,
                    type=type,
                    tag=tag,
                    sort=sort
                )
    pool.join()

    return list(video_douban_ids) 

# 需要导入模块: import gevent [as 别名]
# 或者: from gevent import pool [as 别名]
def send(r, pool=None, stream=False):
    if pool != None:
        return pool.spawn(r.send, stream=stream)
    return gevent.spawn(r.send, stream=stream) 

# 需要导入模块: import gevent [as 别名]
# 或者: from gevent import pool [as 别名]
def __init__(self,options):
		self.options = options
		self.blockip = None
		self.keywords = False
		self.pool = Pool(self.options['threads_count'])
		self.document = self.options['target'].replace('.','_')+'.txt'
		socket.setdefaulttimeout(self.options['timeout'])

	# 域名解析 

# 需要导入模块: import gevent [as 别名]
# 或者: from gevent import pool [as 别名]
def __init__(self, *args, **kwargs):
        pool_size = self.DEFAULT_POOL_SIZE
        if "pool_size" in kwargs:
            pool_size = kwargs.pop("pool_size")
        self.gevent_pool = gevent.pool.Pool(pool_size)
        self.children = []
        self.gevent_worker = None
        super(GeventWorker, self).__init__(*args, **kwargs) 

# 需要导入模块: import gevent [as 别名]
# 或者: from gevent import pool [as 别名]
def dequeue_job_and_maintain_ttl(self, timeout):
        if self._stop_requested:
            raise StopRequested()

        result = None
        while True:
            if self._stop_requested:
                raise StopRequested()

            self.heartbeat()

            if self.gevent_pool.full():
                self.set_state(WorkerStatus.BUSY)
                self.log.warning(
                    "RQ GEVENT worker greenlet pool empty current size %s",
                    self.gevent_pool.size,
                )

            while self.gevent_pool.full():
                gevent.sleep(0.1)
                if self._stop_requested:
                    raise StopRequested()

            try:
                result = self.queue_class.dequeue_any(
                    self.queues, timeout, connection=self.connection
                )
                self.set_state(WorkerStatus.IDLE)
                if result is not None:
                    job, queue = result
                    self.log.info(
                        "%s: %s (%s)"
                        % (green(queue.name), blue(job.description), job.id)
                    )
                break
            except DequeueTimeout:
                pass

        self.heartbeat()
        return result 

# 需要导入模块: import gevent [as 别名]
# 或者: from gevent import pool [as 别名]
def main(self):
        spawny = self.spawn_list()
        if self.kombu:
            gevent.spawn(self.run).join()
        try:
            gevent.pool.joinall(spawny)
        except LoopExit:
            logger.error('No tasks. This operation would block forever.')
        # print([worker.get() for worker in spawny])
        gevent.sleep(0)