Python ThreadPool.join方法代码示例

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import join [as 别名]
    def run_proxy_parse(self, url, proxy_type):
        try:
            for i in range(3):
                #这里对url进行重新组装
                url = url[0:-1] + str(i+1)
                resp = requests.get(url, timeout=5)
                response = etree.HTML(resp.content.decode('utf-8'))
                #解析代理IP，获取代理ip列表
                proxy_ip_list = []
                for item in response.xpath('//tr[contains(@style,"font-size")]'):
                    ip = item.xpath('./td[1]/text()')[0].strip()
                    port = item.xpath('./td[2]/text()')[0].strip()
                    proto = item.xpath('./td[4]/text()')[0].strip()
                    if proto.lower() == 'https':
                        continue
                    proxy_ip = proto.lower() + '://' + ip + ':' + port
                    if proxy_ip not in proxy_ip_list:
                        proxy_ip_list.append(proxy_ip)

                #对每个IP进行测试，留下可用的代理ip
                pool = ThreadPool(len(proxy_ip_list))
                for pip in proxy_ip_list:
                    pool.spawn(self.test_single_proxy, pip, self.test_urls[proxy_type.lower()])
                pool.join()
            
        except Exception, e:
            pass

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import join [as 别名]
class Worker(object):
    def __init__(self, url):
        self.url = url
        self.headers = {'content-type': 'application/json'}
        self.id = 0
        self.post = functools.partial(requests.post, self.url, headers=self.headers)
        self.pool = ThreadPool(100)

    def _call(self, method, *args):
        payload = dict(method=method, params=args, jsonrpc="2.0", id=self.id)
        self.id += 1
        response = self.post(json.dumps(payload)).json()
        return response

    def _async_call(self, method, *args):
        payload = dict(method=method, params=args, jsonrpc="2.0", id=self.id)
        self.id += 1

        def _delayed_call(pl):
            return self.post(json.dumps(pl)).json()

        return Future(self.pool.spawn(_delayed_call, payload))

    def tell(self, method, *args):
        self._async_call(method, *args)

    def ask(self, method, *args):
        return self._async_call(method, *args)

    def join(self):
        self.pool.join()

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import join [as 别名]
class TestJoinEmpty(TestCase):
    switch_expected = False

    def test(self):
        self.pool = ThreadPool(1)
        self.pool.join()

    def cleanup(self):
        # needed here because currently Greenlet.kill() switches out even if greenlet not started yet
        # XXX fix Greenlet.kill
        pass

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import join [as 别名]
def bench_apply(loops):
    pool = ThreadPool(1)
    t0 = perf.perf_counter()

    for _ in xrange(loops):
        for _ in xrange(N):
            pool.apply(noop)

    pool.join()
    pool.kill()
    return perf.perf_counter() - t0

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import join [as 别名]
def bench_spawn_wait(loops):
    pool = ThreadPool(1)

    t0 = perf.perf_counter()

    for _ in xrange(loops):
        for _ in xrange(N):
            r = pool.spawn(noop)
            r.get()

    pool.join()
    pool.kill()
    return perf.perf_counter() - t0

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import join [as 别名]
def my_task(self):
    min_pool = ThreadPool(2)

    self.update_state(state='PROGRESS',
                      meta={'current': 0, 'total': 2,
                            'status': 'PENDING'})

    # spawn all the threads
    greenlets = [min_pool.spawn(_consumer, i) for i in xrange(100)]
    # wait till the finish
    min_pool.join()

    # get the result
    response = [greenlet.get() for greenlet in greenlets]
    return {'current': 100, 'total': 100, 'status': 'Task completed!',
            'result': len(response)}

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import join [as 别名]
    def test_auto_timeout_client__short_timeout_on_stuck_server(self):
        import time
        from threading import Event

        wait_for_start = Event()
        wait_for_close = Event()

        def thread_server(wait_for_start, wait_for_close):
            try:
                print(("starting server, hub: {}".format(gevent.hub.get_hub())))
                with logbook.NullHandler().applicationbound():
                    with server_context(FooService(), max_response_time=0.1):
                        print("server started.")
                        wait_for_start.set()
                        while not wait_for_close.is_set():
                            gevent.sleep(0.1)
            except:
                import traceback
                traceback.print_exc()

        from gevent.threadpool import ThreadPool

        t = ThreadPool(1)
        t.size = 1
        t.spawn(thread_server, wait_for_start, wait_for_close)

        try:
            print(("starting client, hub: {}".format(gevent.hub.get_hub())))
            client = AutoTimeoutClient(ZeroRPCClientTransport.create_tcp(8192), timeout_calc_func=lambda n: n * 2)
            wait_for_start.wait()
            print("client started.")
            t1 = time.time()
            self.assertRaises(TimeoutExpired, client.stuck_call)
            t2 = time.time()
            # This test should always pass although we're dealing with timing and non-deterministic measures since
            # stuck_call() is stuck for an entire second while we're comparing time to 0.2 (almost an order of a
            # magnitude)
            self.assertAlmostEqual(0.2, t2 - t1, delta=0.2)
        finally:
            wait_for_close.set()
            t.join()

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import join [as 别名]
class TestPool(TestCase):
    __timeout__ = 5
    size = 1

    def setUp(self):
        greentest.TestCase.setUp(self)
        self.pool = ThreadPool(self.size)

    def test_apply(self):
        papply = self.pool.apply
        self.assertEqual(papply(sqr, (5,)), sqr(5))
        self.assertEqual(papply(sqr, (), {'x': 3}), sqr(x=3))

    def test_map(self):
        pmap = self.pool.map
        self.assertEqual(pmap(sqr, range(10)), list(map(sqr, range(10))))
        self.assertEqual(pmap(sqr, range(100)), list(map(sqr, range(100))))

    def test_async(self):
        res = self.pool.apply_async(sqr, (7, TIMEOUT1,))
        get = TimingWrapper(res.get)
        self.assertEqual(get(), 49)
        self.assertAlmostEqual(get.elapsed, TIMEOUT1, 1)

    def test_async_callback(self):
        result = []
        res = self.pool.apply_async(sqr, (7, TIMEOUT1,), callback=lambda x: result.append(x))
        get = TimingWrapper(res.get)
        self.assertEqual(get(), 49)
        self.assertAlmostEqual(get.elapsed, TIMEOUT1, 1)
        gevent.sleep(0)  # let's the callback run
        assert result == [49], result

    def test_async_timeout(self):
        res = self.pool.apply_async(sqr, (6, TIMEOUT2 + 0.2))
        get = TimingWrapper(res.get)
        self.assertRaises(gevent.Timeout, get, timeout=TIMEOUT2)
        self.assertAlmostEqual(get.elapsed, TIMEOUT2, 1)
        self.pool.join()

    def test_imap(self):
        it = self.pool.imap(sqr, range(10))
        self.assertEqual(list(it), list(map(sqr, range(10))))

        it = self.pool.imap(sqr, range(10))
        for i in range(10):
            self.assertEqual(six.advance_iterator(it), i * i)
        self.assertRaises(StopIteration, lambda: six.advance_iterator(it))

        it = self.pool.imap(sqr, range(1000))
        for i in range(1000):
            self.assertEqual(six.advance_iterator(it), i * i)
        self.assertRaises(StopIteration, lambda: six.advance_iterator(it))

    def test_imap_random(self):
        it = self.pool.imap(sqr_random_sleep, range(10))
        self.assertEqual(list(it), list(map(sqr, range(10))))

    def test_imap_unordered(self):
        it = self.pool.imap_unordered(sqr, range(1000))
        self.assertEqual(sorted(it), list(map(sqr, range(1000))))

        it = self.pool.imap_unordered(sqr, range(1000))
        self.assertEqual(sorted(it), list(map(sqr, range(1000))))

    def test_imap_unordered_random(self):
        it = self.pool.imap_unordered(sqr_random_sleep, range(10))
        self.assertEqual(sorted(it), list(map(sqr, range(10))))

    def test_terminate(self):
        result = self.pool.map_async(sleep, [0.1] * ((self.size or 10) * 2))
        gevent.sleep(0.1)
        kill = TimingWrapper(self.pool.kill)
        kill()
        assert kill.elapsed < 0.5, kill.elapsed
        result.join()

    def sleep(self, x):
        sleep(float(x) / 10.)
        return str(x)

    def test_imap_unordered_sleep(self):
        # testing that imap_unordered returns items in competion order
        result = list(self.pool.imap_unordered(self.sleep, [10, 1, 2]))
        if self.pool.size == 1:
            expected = ['10', '1', '2']
        else:
            expected = ['1', '2', '10']
        self.assertEqual(result, expected)

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import join [as 别名]
class TestJoinEmpty(TestCase):
    switch_expected = False

    def test(self):
        self.pool = ThreadPool(1)
        self.pool.join()

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import join [as 别名]
        test = importlib.import_module(task[:task.find('.py')])
        print '========================================================================'
        print 'Starting Test:', test.TEST['name']
        if complete_task(test) is False:
            print test.TEST['name'], 'Test Failed'
            return

        print test.TEST['name'], 'Test Success'


if __name__ == '__main__':
    get_input()

    test_suite = importlib.import_module(input_file[:input_file.find('.py')])
    print '========================================================================'
    print 'Test Suite Project Name:', test_suite.TEST_ENV['project_name']

    if 'init_hooks' in test_suite.TEST_ENV:
        if not test_suite.TEST_ENV['init_hooks'](global_headers, global_post_param, global_query_param):
            print 'Init Failed'
    domain = '%s://%s' % (test_suite.TEST_ENV['protocol'], test_suite.TEST_ENV['domain'])

    for i in range(0, len(test_suite.TEST_ENV['testcases'])):
        tasks.put_nowait(test_suite.TEST_ENV['testcases'][i])

    pool = ThreadPool(threads_count)
    for _ in range(threads_count):
        pool.spawn(worker)

    pool.join()

# 需要导入模块: from gevent.threadpool import ThreadPool [as 别名]
# 或者: from gevent.threadpool.ThreadPool import join [as 别名]
def download(url, output, 
        thread_count = defaults['thread_count'], 
        buffer_size = defaults['buffer_size'], 
        block_size = defaults['block_size']):
    # get latest file info
    file_info = get_file_info(url)

    # init path
    if output is None:
        output = file_info.name
    workpath = '%s.ing' % output
    infopath = '%s.inf' % output

    # split file to blocks. every block is a array [start, offset, end],
    # then each greenlet download filepart according to a block, and 
    # update the block' offset.
    blocks = []

    if os.path.exists(infopath):
        # load blocks
        _x, blocks = read_data(infopath)        
        if (_x.url != url or 
                _x.name != file_info.name or 
                _x.lastmodified != file_info.lastmodified):
            blocks = []

    if len(blocks) == 0:
        # set blocks
        if block_size > file_info.size:
            blocks = [[0, 0, file_info.size]]
        else:
            block_count, remain = divmod(file_info.size, block_size)
            blocks = [[i*block_size, i*block_size, (i+1)*block_size-1] for i in range(block_count)]
            blocks[-1][-1] += remain
        # create new blank workpath
        with open(workpath, 'wb') as fobj:
            fobj.write('')

    # start monitor
    monitor = gevent.spawn(_monitor, infopath, file_info, blocks)
    
    # start downloading
    with open(workpath, 'rb+') as fobj:
        args = [(url, blocks[i], fobj, buffer_size) for i in range(len(blocks)) if blocks[i][1] < blocks[i][2]]

        if thread_count > len(args):
            thread_count = len(args)

        pool = ThreadPool(thread_count)
        pool.map(_worker, args)
        pool.join()

    monitor.join()

    # rename workpath to output
    if os.path.exists(output):
        os.remove(output)
    os.rename(workpath, output)

    # delete infopath
    if os.path.exists(infopath):
        os.remove(infopath)

    print 'thread_count ', thread_count