Python process.ProcessPoolExecutor方法代码示例

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import ProcessPoolExecutor [as 别名]
def main():
    urls = [
        'https://talkpython.fm',
        'https://pythonbytes.fm',
        'https://google.com',
        'https://realpython.com',
        'https://training.talkpython.fm/',
    ]

    work = []

    with PoolExecutor() as executor:
        for url in urls:
            # print("Getting title from {}".format(url.replace('https', '')),
            #       end='... ',
            #       flush=True)
            # title = get_title(url)
            f: Future = executor.submit(get_title, url)
            work.append(f)

        print("Waiting for downloads...", flush=True)

    print("Done", flush=True)
    for f in work:
        print("{}".format(f.result()), flush=True) 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import ProcessPoolExecutor [as 别名]
def musicSpider():
    print("======= 开始爬 音乐 信息 ===========")
    startTime = datetime.datetime.now()
    print(startTime.strftime('%Y-%m-%d %H:%M:%S'))
    # 所有专辑数量
    albums_num = sql.get_all_album_num()
    print("所有专辑数量：", albums_num)
    # 批次
    batch = math.ceil(albums_num.get('num') / 1000.0)
    # 构建线程池
    # pool = ProcessPoolExecutor(1)
    for index in range(0, batch):
        saveMusicBatch(index)
        # pool.submit(saveMusicBatch, index)
    # pool.shutdown(wait=True)
    print("======= 结束爬 音乐 信息 ===========")
    endTime = datetime.datetime.now()
    print(endTime.strftime('%Y-%m-%d %H:%M:%S'))
    print("耗时：", (endTime - startTime).seconds, "秒")

# if __name__ == '__main__':
#     musicSpider() 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import ProcessPoolExecutor [as 别名]
def lyricSpider():
    print("======= 开始爬 歌词 信息 ===========")
    startTime = datetime.datetime.now()
    print(startTime.strftime('%Y-%m-%d %H:%M:%S'))
    # 所有歌手数量
    musics_num = sql.get_all_music_num()
    # 批次
    batch = math.ceil(musics_num.get('num') / 1000.0)
    # 构建线程池
    # pool = ProcessPoolExecutor(1)
    for index in range(0, batch):
        saveLyricBatch(index)
        # pool.submit(saveLyricBatch, index)
    # pool.shutdown(wait=True)
    print("======= 结束爬 歌词 信息 ===========")
    endTime = datetime.datetime.now()
    print(endTime.strftime('%Y-%m-%d %H:%M:%S'))
    print("耗时：", (endTime - startTime).seconds, "秒")

# if __name__ == '__main__':
#     lyricSpider() 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import ProcessPoolExecutor [as 别名]
def commentSpider():
    print("======= 开始爬 评论 信息 ===========")
    startTime = datetime.datetime.now()
    print(startTime.strftime('%Y-%m-%d %H:%M:%S'))
    # 所有歌手数量
    musics_num = sql.get_all_music_num()
    # 批次
    batch = math.ceil(musics_num.get('num') / 1000.0)
    # 构建线程池
    # pool = ProcessPoolExecutor(1)
    for index in range(0, batch):
        saveCommentBatch(index)
        # pool.submit(saveCommentBatch, index)
    # pool.shutdown(wait=True)
    print("======= 结束爬 评论 信息 ===========")
    endTime = datetime.datetime.now()
    print(endTime.strftime('%Y-%m-%d %H:%M:%S'))
    print("耗时：", (endTime - startTime).seconds, "秒")

# if __name__ == '__main__':
#     commentSpider() 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import ProcessPoolExecutor [as 别名]
def test_init_cpu(dist_init_method):
    args = argparse.Namespace(
        multiprocessing_distributed=True,
        dist_world_size=2,
        dist_rank=None,
        ngpu=0,
        local_rank=None,
        dist_launcher=None,
        distributed=True,
        dist_backend="gloo",
        dist_init_method=dist_init_method,
        dist_master_addr=None,
        dist_master_port=None,
    )
    args.dist_rank = 0
    option = build_dataclass(DistributedOption, args)
    args.dist_rank = 1
    option2 = build_dataclass(DistributedOption, args)
    with ProcessPoolExecutor(max_workers=2) as e:
        fn = e.submit(option.init)
        fn2 = e.submit(option2.init)
        fn.result()
        fn2.result() 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import ProcessPoolExecutor [as 别名]
def test_init_cpu2():
    args = argparse.Namespace(
        multiprocessing_distributed=True,
        dist_world_size=2,
        dist_rank=None,
        ngpu=0,
        local_rank=None,
        dist_launcher=None,
        distributed=True,
        dist_backend="gloo",
        dist_init_method="env://",
        dist_master_addr=None,
        dist_master_port=free_port(),
    )
    args.dist_rank = 0
    option = build_dataclass(DistributedOption, args)
    args.dist_rank = 1
    option2 = build_dataclass(DistributedOption, args)
    with ProcessPoolExecutor(max_workers=2) as e:
        fn = e.submit(option.init)
        fn2 = e.submit(option2.init)
        with pytest.raises(RuntimeError):
            fn.result()
            fn2.result() 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import ProcessPoolExecutor [as 别名]
def test_init_cpu4():
    args = argparse.Namespace(
        multiprocessing_distributed=True,
        dist_world_size=2,
        dist_rank=None,
        ngpu=0,
        local_rank=None,
        dist_launcher=None,
        distributed=True,
        dist_backend="gloo",
        dist_init_method="env://",
        dist_master_addr="localhost",
        dist_master_port=free_port(),
    )
    args.dist_rank = 0
    option = build_dataclass(DistributedOption, args)
    args.dist_rank = 1
    option2 = build_dataclass(DistributedOption, args)
    with ProcessPoolExecutor(max_workers=2) as e:
        fn = e.submit(option.init)
        fn2 = e.submit(option2.init)
        fn.result()
        fn2.result() 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import ProcessPoolExecutor [as 别名]
def test_init_cpu5():
    args = argparse.Namespace(
        multiprocessing_distributed=True,
        dist_world_size=2,
        dist_rank=None,
        ngpu=0,
        local_rank=None,
        dist_launcher=None,
        distributed=True,
        dist_backend="gloo",
        dist_init_method="env://",
        dist_master_addr="localhost",
        dist_master_port=free_port(),
    )
    args.dist_rank = 0
    option = build_dataclass(DistributedOption, args)
    args.dist_rank = 1
    option2 = build_dataclass(DistributedOption, args)
    with ProcessPoolExecutor(max_workers=2) as e:
        fn = e.submit(option.init)
        fn2 = e.submit(option2.init)
        fn.result()
        fn2.result() 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import ProcessPoolExecutor [as 别名]
def process_all(self, texts: Collection[str]) -> List[List[str]]:
        "Process a list of `texts`."
        if self.n_cpus <= 1:
            return self._process_all_1(texts)
        with ProcessPoolExecutor(self.n_cpus) as e:
            return sum(
                e.map(self._process_all_1, partition_by_cores(texts, self.n_cpus)), []
            ) 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import ProcessPoolExecutor [as 别名]
def __init__(self, num_workers):
        '''
            Create a process pool for concurrent execution with specified number of workers.
        '''
        self.executor_pool = ProcessPoolExecutor(num_workers) 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import ProcessPoolExecutor [as 别名]
def __init__(self, executor: ProcessPoolExecutor):
        self._executor = executor
        self._loop = asyncio.get_event_loop() 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import ProcessPoolExecutor [as 别名]
def main(*, host: str = '127.0.0.1', port: int = 50051) -> None:
    with ProcessPoolExecutor(max_workers=4) as executor:
        server = Server([Primes(executor)])
        with graceful_exit([server]):
            await server.start(host, port)
            print(f'Serving on {host}:{port}')
            await server.wait_closed() 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import ProcessPoolExecutor [as 别名]
def test_resolve_distributed_mode_slurm3():
    args = argparse.Namespace(
        multiprocessing_distributed=True,
        dist_world_size=None,
        dist_rank=None,
        ngpu=1,
        local_rank=None,
        dist_launcher="slurm",
        dist_backend="nccl",
        dist_init_method="env://",
        dist_master_addr=None,
        dist_master_port=10000,
    )
    env = dict(
        SLURM_PROCID="0",
        SLURM_NTASKS="1",
        SLURM_STEP_NUM_NODES="1",
        SLURM_STEP_NODELIST="localhost",
        SLURM_NODEID="0",
        CUDA_VISIBLE_DEVICES="0,1",
    )

    e = ProcessPoolExecutor(max_workers=2)
    with unittest.mock.patch.dict("os.environ", dict(env, SLURM_LOCALID="0")):
        resolve_distributed_mode(args)
        option = build_dataclass(DistributedOption, args)
        fn = e.submit(option.init)

    with unittest.mock.patch.dict("os.environ", dict(env, SLURM_LOCALID="0")):
        option2 = build_dataclass(DistributedOption, args)
        fn2 = e.submit(option2.init)

    fn.result()
    fn2.result() 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import ProcessPoolExecutor [as 别名]
def __init__(self, max_workers=None):
        if max_workers is None:
            max_workers = _utils.get_optimal_process_count()
        super(ProcessPoolExecutor, self).__init__(max_workers=max_workers)
        if self._max_workers <= 0:
            raise ValueError("Max workers must be greater than zero")
        self._gatherer = _Gatherer(
            # Since our submit will use this gatherer we have to reference
            # the parent submit, bound to this instance (which is what we
            # really want to use anyway).
            super(ProcessPoolExecutor, self).submit,
            self.threading.lock_object) 

# 需要导入模块: from concurrent.futures import process [as 别名]
# 或者: from concurrent.futures.process import ProcessPoolExecutor [as 别名]
def _set_executor_pool(self) -> None:
        """Set thread pool pool to be used."""
        self._executor_pool = ProcessPoolExecutor(max_workers=len(self._tasks))