Python futurist.ThreadPoolExecutor方法代码示例

# 需要导入模块: import futurist [as 别名]
# 或者: from futurist import ThreadPoolExecutor [as 别名]
def test_ngs_basic_dlm_ops(self):
        pool = futurist.ThreadPoolExecutor()
        self.addCleanup(pool.shutdown)
        fts = []
        for i in range(CONF.ngs.port_dlm_concurrency):
            fts.append(
                pool.submit(
                    self._test_ngs_basic_ops,
                    port_name='{base}_{ind}'.format(
                        base=CONF.ngs.port_name, ind=i)))

        executed = futurist.waiters.wait_for_all(fts)
        self.assertFalse(executed.not_done)
        # TODO(pas-ha) improve test error reporting here
        for ft in executed.done:
            self.assertIsNone(ft.exception()) 

# 需要导入模块: import futurist [as 别名]
# 或者: from futurist import ThreadPoolExecutor [as 别名]
def __init__(self, exchange, topic, tasks,
                 executor=None, threads_count=None, url=None,
                 transport=None, transport_options=None,
                 retry_options=None):
        self._topic = topic
        self._executor = executor
        self._owns_executor = False
        if self._executor is None:
            self._executor = futurist.ThreadPoolExecutor(
                max_workers=threads_count)
            self._owns_executor = True
        self._endpoints = self._derive_endpoints(tasks)
        self._exchange = exchange
        self._server = server.Server(topic, exchange, self._executor,
                                     self._endpoints, url=url,
                                     transport=transport,
                                     transport_options=transport_options,
                                     retry_options=retry_options) 

# 需要导入模块: import futurist [as 别名]
# 或者: from futurist import ThreadPoolExecutor [as 别名]
def test_execution_concurrency_scale_up(self):
        package = self.create_package(name='python/test_python_sleep.py')
        function_id = self.create_function(package_path=package)

        def _create_execution():
            resp, body = self.client.create_execution(function_id)
            return resp, body

        futs = []
        with futurist.ThreadPoolExecutor(max_workers=10) as executor:
            for _ in range(6):
                fut = executor.submit(_create_execution)
                futs.append(fut)
            for f in futures.as_completed(futs):
                # Wait until we get the response
                resp, body = f.result()

                self.assertEqual(201, resp.status)
                self.addCleanup(self.client.delete_resource, 'executions',
                                body['id'], ignore_notfound=True)
                self.assertEqual('success', body['status'])

        resp, body = self.admin_client.get_function_workers(function_id)
        self.assertEqual(200, resp.status)
        self.assertEqual(2, len(body['workers'])) 

# 需要导入模块: import futurist [as 别名]
# 或者: from futurist import ThreadPoolExecutor [as 别名]
def __init__(self, worker_id):
        super(StressNotificationsService, self).__init__(worker_id)
        self.oslo_notifier = None
        topics = CONF.datasources.notification_topics
        self.oslo_notifier = oslo_messaging.Notifier(
            get_transport(),
            driver='messagingv2',
            publisher_id='vitrage.stress',
            topics=topics)
        self.periodic = periodics.PeriodicWorker.create(
            [], executor_factory=lambda: ThreadPoolExecutor(max_workers=10)) 

# 需要导入模块: import futurist [as 别名]
# 或者: from futurist import ThreadPoolExecutor [as 别名]
def _do_collection(self, metrics, timestamp):

        def _get_result(metric):
            try:
                return self._collect(metric, timestamp)
            except collector.NoDataCollected:
                LOG.info(
                    self._log_prefix + 'No data collected '
                    'for metric {metric} at timestamp {ts}'.format(
                        metric=metric, ts=timestamp))
                return metric, None
            except Exception as e:
                LOG.exception(
                    self._log_prefix + 'Error while collecting'
                    ' metric {metric} at timestamp {ts}: {e}. Exiting.'.format(
                        metric=metric, ts=timestamp, e=e))
                # FIXME(peschk_l): here we just exit, and the
                # collection will be retried during the next collect
                # cycle. In the future, we should implement a retrying
                # system in workers
                sys.exit(1)

        with futurist.ThreadPoolExecutor(
                max_workers=CONF.orchestrator.max_threads) as tpool:
            futs = [tpool.submit(_get_result, metric) for metric in metrics]
            LOG.debug(self._log_prefix +
                      'Collecting {} metrics.'.format(len(metrics)))
            results = [r.result() for r in waiters.wait_for_all(futs).done]
            LOG.debug(self._log_prefix + 'Collecting {} metrics took {}s '
                      'total, with {}s average'.format(
                          tpool.statistics.executed,
                          tpool.statistics.runtime,
                          tpool.statistics.average_runtime))
        return dict(filter(lambda x: x[1] is not None, results)) 

# 需要导入模块: import futurist [as 别名]
# 或者: from futurist import ThreadPoolExecutor [as 别名]
def _create_executor(self, max_workers=None):
        return futurist.ThreadPoolExecutor(max_workers=max_workers) 

# 需要导入模块: import futurist [as 别名]
# 或者: from futurist import ThreadPoolExecutor [as 别名]
def _default_executor_factory(self):
        max_simultaneous_jobs = self._max_simultaneous_jobs
        if max_simultaneous_jobs <= 0:
            max_workers = tu.get_optimal_thread_count()
        else:
            max_workers = max_simultaneous_jobs
        return futurist.ThreadPoolExecutor(max_workers=max_workers) 

# 需要导入模块: import futurist [as 别名]
# 或者: from futurist import ThreadPoolExecutor [as 别名]
def test_using_common_executor(self):
        flow = utils.TaskNoRequiresNoReturns(name='task1')
        executor = futurist.ThreadPoolExecutor(self._EXECUTOR_WORKERS)
        try:
            e1 = self._make_engine(flow, executor=executor)
            e2 = self._make_engine(flow, executor=executor)
            self.assertIs(e1.options['executor'], e2.options['executor'])
        finally:
            executor.shutdown(wait=True) 

# 需要导入模块: import futurist [as 别名]
# 或者: from futurist import ThreadPoolExecutor [as 别名]
def single_factory():
    return futurist.ThreadPoolExecutor(max_workers=1) 

# 需要导入模块: import futurist [as 别名]
# 或者: from futurist import ThreadPoolExecutor [as 别名]
def _fetch_server(self, task_classes):
        endpoints = []
        for cls in task_classes:
            endpoints.append(endpoint.Endpoint(cls))
        server = worker_server.Server(
            TEST_TOPIC, TEST_EXCHANGE,
            futurist.ThreadPoolExecutor(max_workers=1), endpoints,
            transport='memory',
            transport_options={
                'polling_interval': POLLING_INTERVAL,
            })
        server_thread = threading_utils.daemon_thread(server.start)
        return (server, server_thread) 

# 需要导入模块: import futurist [as 别名]
# 或者: from futurist import ThreadPoolExecutor [as 别名]
def test_execution_concurrency_no_scale(self):
        package = self.create_package(name='python/test_python_sleep.py')
        function_id = self.create_function(package_path=package)

        def _create_execution():
            resp, body = self.client.create_execution(function_id)
            return resp, body

        futs = []
        with futurist.ThreadPoolExecutor(max_workers=10) as executor:
            for _ in range(3):
                fut = executor.submit(_create_execution)
                futs.append(fut)
            for f in futures.as_completed(futs):
                # Wait until we get the response
                resp, body = f.result()

                self.assertEqual(201, resp.status)
                self.addCleanup(self.client.delete_resource, 'executions',
                                body['id'], ignore_notfound=True)
                self.assertEqual('success', body['status'])

        resp, body = self.admin_client.get_function_workers(function_id)

        self.assertEqual(200, resp.status)
        self.assertEqual(1, len(body['workers'])) 

# 需要导入模块: import futurist [as 别名]
# 或者: from futurist import ThreadPoolExecutor [as 别名]
def main():
    if len(sys.argv) == 2:
        tbl = []
        with open(sys.argv[1], 'rb') as fh:
            reader = csv.reader(fh)
            for row in reader:
                tbl.append([float(r) if r else 0.0 for r in row])
    else:
        # Make some random table out of thin air...
        tbl = []
        cols = random.randint(1, 100)
        rows = random.randint(1, 100)
        for _i in compat_range(0, rows):
            row = []
            for _j in compat_range(0, cols):
                row.append(random.random())
            tbl.append(row)

    # Generate the work to be done.
    f = make_flow(tbl)

    # Now run it (using the specified executor)...
    try:
        executor = futurist.GreenThreadPoolExecutor(max_workers=5)
    except RuntimeError:
        # No eventlet currently active, use real threads instead.
        executor = futurist.ThreadPoolExecutor(max_workers=5)
    try:
        e = engines.load(f, engine='parallel', executor=executor)
        for st in e.run_iter():
            print(st)
    finally:
        executor.shutdown()

    # Find the old rows and put them into place...
    #
    # TODO(harlowja): probably easier just to sort instead of search...
    computed_tbl = []
    for i in compat_range(0, len(tbl)):
        for t in f:
            if t.index == i:
                computed_tbl.append(e.storage.get(t.name))

    # Do some basic validation (which causes the return code of this process
    # to be different if things were not as expected...)
    if len(computed_tbl) != len(tbl):
        return 1
    else:
        return 0 

# 需要导入模块: import futurist [as 别名]
# 或者: from futurist import ThreadPoolExecutor [as 别名]
def connect(self, timeout=10.0):

        def try_clean():
            # Attempt to do the needed cleanup if post-connection setup does
            # not succeed (maybe the connection is lost right after it is
            # obtained).
            try:
                self.close()
            except k_exceptions.KazooException:
                LOG.exception("Failed cleaning-up after post-connection"
                              " initialization failed")

        try:
            if timeout is not None:
                timeout = float(timeout)
            self._client.start(timeout=timeout)
            self._closing = False
        except (self._client.handler.timeout_exception,
                k_exceptions.KazooException):
            excp.raise_with_cause(excp.JobFailure,
                                  "Failed to connect to zookeeper")
        try:
            if self._conf.get('check_compatible', True):
                kazoo_utils.check_compatible(self._client, self.MIN_ZK_VERSION)
            if self._worker is None and self._emit_notifications:
                self._worker = futurist.ThreadPoolExecutor(max_workers=1)
            self._client.ensure_path(self.path)
            self._client.ensure_path(self.trash_path)
            if self._job_watcher is None:
                self._job_watcher = watchers.ChildrenWatch(
                    self._client,
                    self.path,
                    func=self._on_job_posting,
                    allow_session_lost=True)
            self._connected = True
        except excp.IncompatibleVersion:
            with excutils.save_and_reraise_exception():
                try_clean()
        except (self._client.handler.timeout_exception,
                k_exceptions.KazooException):
            exc_type, exc, exc_tb = sys.exc_info()
            try:
                try_clean()
                excp.raise_with_cause(excp.JobFailure,
                                      "Failed to do post-connection"
                                      " initialization", cause=exc)
            finally:
                del(exc_type, exc, exc_tb)