Python thread.ThreadPoolExecutor方法代码示例

# 需要导入模块: from concurrent.futures import thread [as 别名]
# 或者: from concurrent.futures.thread import ThreadPoolExecutor [as 别名]
def __init__(self):
        """初始化事件管理器"""
        # 事件对象列表
        self.__eventQueue = Queue()
        # 事件管理器开关
        self.__active = False
        # 事件处理线程
        self.__thread = Thread(target=self.__run, name='Synchronous')
        # 事件处理线程池
        self.__pool = ThreadPoolExecutor(3)
        # 阻塞函数列表
        self.__block = []

        # 这里的__handlers是一个字典，用来保存对应的事件的响应函数
        # 其中每个键对应的值是一个列表，列表中保存了对该事件监听的响应函数，一对多
        self.__handlers = {}

        self.__method = {} 

# 需要导入模块: from concurrent.futures import thread [as 别名]
# 或者: from concurrent.futures.thread import ThreadPoolExecutor [as 别名]
def progressbar(self, label: str, total: int):
        bar = progressbar(
            length=total,
            fill_char=stream.green(self.BAR_FILLED_CHAR),
            empty_char=self.BAR_EMPTY_CHAR,
            show_percent=False,
            show_pos=True,
            label=label,
            bar_template="%(label)s %(bar)s %(info)s",
        )
        if self.parallel:
            executor = ThreadPoolExecutor()
        else:
            executor = DummyExecutor()
        with executor:
            try:
                yield bar, executor
            except KeyboardInterrupt:
                pass 

# 需要导入模块: from concurrent.futures import thread [as 别名]
# 或者: from concurrent.futures.thread import ThreadPoolExecutor [as 别名]
def run(self):
        loop = asyncio.get_event_loop()
        executor = ThreadPoolExecutor(1)
        loop.set_default_executor(executor)

        def __exit():
            raise SystemExit()
        try:
            loop.add_signal_handler(signal.SIGINT, __exit)
            loop.add_signal_handler(signal.SIGTERM, __exit)
            loop.run_until_complete(self.start())
            loop.run_until_complete(self.shutdown_event.wait())
        except (SystemExit, KeyboardInterrupt):
            pass
        finally:
            loop.run_until_complete(self.stop())
            executor.shutdown(True) 

# 需要导入模块: from concurrent.futures import thread [as 别名]
# 或者: from concurrent.futures.thread import ThreadPoolExecutor [as 别名]
def test_cancel(self, cursor):
        def cancel(c):
            time.sleep(randint(5, 10))
            c.cancel()

        with ThreadPoolExecutor(max_workers=1) as executor:
            executor.submit(cancel, cursor)

            self.assertRaises(
                DatabaseError,
                lambda: cursor.execute(
                    """
                    SELECT a.a * rand(), b.a * rand()
                    FROM many_rows a
                    CROSS JOIN many_rows b
                    """
                ),
            ) 

# 需要导入模块: from concurrent.futures import thread [as 别名]
# 或者: from concurrent.futures.thread import ThreadPoolExecutor [as 别名]
def create_pool(self, job, parameters, workers=None):
        """ Create resources in pool in sub processes.

        :param workers:
        :type parameters: iterable
        :type job: func
        """
        executor = ThreadPoolExecutor(
            workers) if workers else ThreadPoolExecutor()
        try:
            # futures = [executor.submit(func, i, kwargs) for i in args]
            futures = []
            for param_chunk in parameters:
                param_chunk['self'] = self
                futures.append(executor.submit(job, param_chunk))
            concurrent.futures.wait(futures, return_when=ALL_COMPLETED)
            responses = {}
            for future in futures:
                result = future.result()
                if result:
                    responses.update(result)
            return responses
        finally:
            executor.shutdown(wait=True) 

# 需要导入模块: from concurrent.futures import thread [as 别名]
# 或者: from concurrent.futures.thread import ThreadPoolExecutor [as 别名]
def assemble_node_lambdas(project_path, bundles_dir):
    from syndicate.core import CONFIG
    project_abs_path = build_path(CONFIG.project_path, project_path)

    _LOG.info('Going to package lambdas starting by path {0}'.format(
        project_abs_path))
    executor = ThreadPoolExecutor(max_workers=5)
    futures = []
    for root, sub_dirs, files in os.walk(project_abs_path):
        for item in files:
            if item.endswith(LAMBDA_CONFIG_FILE_NAME):
                _LOG.info('Going to build artifact in: {0}'.format(root))
                arg = {
                    'item': item,
                    'root': root,
                    'target_folder': bundles_dir
                }
                futures.append(executor.submit(_build_node_artifact, arg))
    for future in concurrent.futures.as_completed(futures):
        _LOG.info(future.result()) 

# 需要导入模块: from concurrent.futures import thread [as 别名]
# 或者: from concurrent.futures.thread import ThreadPoolExecutor [as 别名]
def assemble_python_lambdas(project_path, bundles_dir):
    from syndicate.core import CONFIG
    project_base_folder = os.path.basename(os.path.normpath(project_path))
    project_abs_path = build_path(CONFIG.project_path, project_path)
    _LOG.info('Going to process python project by path: {0}'.format(
        project_abs_path))
    executor = ThreadPoolExecutor(max_workers=5)
    futures = []
    for root, sub_dirs, files in os.walk(project_abs_path):
        for item in files:
            if item.endswith(LAMBDA_CONFIG_FILE_NAME):
                _LOG.info('Going to build artifact in: {0}'.format(root))
                arg = {
                    'item': item,
                    'project_base_folder': project_base_folder,
                    'project_path': project_path,
                    'root': root,
                    'target_folder': bundles_dir
                }
                futures.append(executor.submit(_build_python_artifact, arg))
    concurrent.futures.wait(futures, return_when=ALL_COMPLETED)
    executor.shutdown()
    _LOG.info('Python project was processed successfully') 

# 需要导入模块: from concurrent.futures import thread [as 别名]
# 或者: from concurrent.futures.thread import ThreadPoolExecutor [as 别名]
def test_init_cpu3():
    args = argparse.Namespace(
        multiprocessing_distributed=True,
        dist_world_size=2,
        dist_rank=None,
        ngpu=0,
        local_rank=None,
        dist_launcher=None,
        distributed=True,
        dist_backend="gloo",
        dist_init_method="env://",
        dist_master_addr="localhost",
        dist_master_port=None,
    )
    args.dist_rank = 0
    option = build_dataclass(DistributedOption, args)
    args.dist_rank = 1
    option2 = build_dataclass(DistributedOption, args)
    with ThreadPoolExecutor(max_workers=2) as e:
        fn = e.submit(option.init)
        fn2 = e.submit(option2.init)
        with pytest.raises(RuntimeError):
            fn.result()
            fn2.result() 

# 需要导入模块: from concurrent.futures import thread [as 别名]
# 或者: from concurrent.futures.thread import ThreadPoolExecutor [as 别名]
def search(
        session: Session, login_response, first_name, last_name, middle_name="", birth_date=""
    ) -> List[OeciCase]:
        search_url = URL.search_url()
        node_response = Crawler._fetch_search_page(session, search_url, login_response)
        oeci_search_result = Crawler._search_record(
            session, node_response, search_url, first_name, last_name, middle_name, birth_date
        )
        case_limit = 300
        if len(oeci_search_result.cases) >= case_limit:
            raise ValueError(
                f"Found {len(oeci_search_result.cases)} matching cases, exceeding the limit of {case_limit}. Please add a date of birth to your search."
            )
        else:
            # Parse search results (case detail pages)
            with ThreadPoolExecutor(max_workers=50) as executor:
                oeci_cases: List[OeciCase] = []
                for oeci_case in executor.map(partial(Crawler._read_case, session), oeci_search_result.cases):
                    oeci_cases.append(oeci_case)
            return oeci_cases 

# 需要导入模块: from concurrent.futures import thread [as 别名]
# 或者: from concurrent.futures.thread import ThreadPoolExecutor [as 别名]
def test_connection_keepalive(simple_wsgi_server):
    """Test the connection keepalive works (duh)."""
    session = Session(base_url=simple_wsgi_server['url'])
    pooled = requests.adapters.HTTPAdapter(
        pool_connections=1, pool_maxsize=1000,
    )
    session.mount('http://', pooled)

    def do_request():
        with ExceptionTrap(requests.exceptions.ConnectionError) as trap:
            resp = session.get('info')
            resp.raise_for_status()
        return bool(trap)

    with ThreadPoolExecutor(max_workers=50) as pool:
        tasks = [
            pool.submit(do_request)
            for n in range(1000)
        ]
        failures = sum(task.result() for task in tasks)

    assert not failures 

# 需要导入模块: from concurrent.futures import thread [as 别名]
# 或者: from concurrent.futures.thread import ThreadPoolExecutor [as 别名]
def test_cancel(self, cursor):
        def cancel(c):
            time.sleep(randint(1, 5))
            c.cancel()

        with ThreadPoolExecutor(max_workers=1) as executor:
            executor.submit(cancel, cursor)

            self.assertRaises(
                DatabaseError,
                lambda: cursor.execute(
                    """
                    SELECT a.a * rand(), b.a * rand()
                    FROM many_rows a
                    CROSS JOIN many_rows b
                    """
                ),
            ) 

# 需要导入模块: from concurrent.futures import thread [as 别名]
# 或者: from concurrent.futures.thread import ThreadPoolExecutor [as 别名]
def run_in_many_threads():
    def run_in_many_threads_(async_fn, max_workers=50, executions=200, *args, **kwargs):
        with ThreadPoolExecutor(max_workers=max_workers) as e:
            futures = []
            for _ in range(0, executions):
                futures.append(e.submit(partial(asyncio.run, async_fn(*args, **kwargs))))

            done, _ = wait(futures)

        for future in done:
            if future.exception():
                raise future.exception()

        return done

    return run_in_many_threads_ 

# 需要导入模块: from concurrent.futures import thread [as 别名]
# 或者: from concurrent.futures.thread import ThreadPoolExecutor [as 别名]
def test_threads(anchore_db):
    """
    Test concurrent lease contention and acquisition

    :param anchore_db:
    :return:
    """

    th = []
    t = ThreadPoolExecutor(max_workers=3)
    th.append(t.submit(run_thread_lock_fn, 2))
    th.append(t.submit(run_thread_lock_fn, 5))
    th.append(t.submit(run_thread_lock_fn, 1))

    for thread in th:
        # Wait for completion
        r = thread.result()
        logger.info(('Thread result {}'.format(r))) 

# 需要导入模块: from concurrent.futures import thread [as 别名]
# 或者: from concurrent.futures.thread import ThreadPoolExecutor [as 别名]
def __init__(self, *args, **kwargs):
        # type: (*Any, **Any) -> None
        super(VSphereCheck, self).__init__(*args, **kwargs)
        instance = cast(InstanceConfig, self.instance)
        self.config = VSphereConfig(instance, self.log)

        self.latest_event_query = get_current_datetime()
        self.infrastructure_cache = InfrastructureCache(interval_sec=self.config.refresh_infrastructure_cache_interval)
        self.metrics_metadata_cache = MetricsMetadataCache(
            interval_sec=self.config.refresh_metrics_metadata_cache_interval
        )
        self.api = cast(VSphereAPI, None)
        self.api_rest = cast(VSphereRestAPI, None)
        # Do not override `AgentCheck.hostname`
        self._hostname = None
        self.thread_pool = ThreadPoolExecutor(max_workers=self.config.threads_count)
        self.check_initializations.append(self.initiate_api_connection) 

# 需要导入模块: from concurrent.futures import thread [as 别名]
# 或者: from concurrent.futures.thread import ThreadPoolExecutor [as 别名]
def test_concurrent_insert_using_connection(conn_cnx, db_parameters):
    """Concurrent insert tests using the same connection."""
    try:
        with conn_cnx() as cnx:
            cnx.cursor().execute("""
create or replace warehouse {}
warehouse_type=standard
warehouse_size=small
""".format(db_parameters['name_wh']))
            cnx.cursor().execute("""
CREATE OR REPLACE TABLE {name} (c1 INTEGER, c2 STRING)
""".format(
                name=db_parameters['name']))
            number_of_threads = 5
            metas = []
            for i in range(number_of_threads):
                metas.append({
                    'connection': cnx,
                    'idx': i,
                    'name': db_parameters['name'],
                })
            pool = ThreadPoolExecutor(number_of_threads)
            pool.map(_concurrent_insert_using_connection, metas)
            pool.shutdown()
            cnt = 0
            for _ in cnx.cursor().execute(
                    "SELECT * FROM {name} ORDER BY 1".format(
                        name=db_parameters['name'])):
                cnt += 1
            assert cnt <= number_of_threads, \
                "Number of records should be less than the number of threads"
            assert cnt > 0, \
                "Number of records should be one or more number of threads"
    finally:
        with conn_cnx() as cnx:
            cnx.cursor().execute(
                "drop table if exists {}".format(db_parameters['name']))
            cnx.cursor().execute(
                "drop warehouse if exists {}".format(db_parameters['name_wh']))