Python concurrent.futures方法代码示例

# 需要导入模块: import concurrent [as 别名]
# 或者: from concurrent import futures [as 别名]
def pool(self):
        result = self.parse_html(self.host)
        try:
            with concurrent.futures.ThreadPoolExecutor(max_workers=30) as executor:
                futures = [executor.submit(self.parse_html, i) for i in result]
                for future in concurrent.futures.as_completed(futures, timeout=3):
                    future.result()
        except (EOFError, concurrent.futures._base.TimeoutError):
            pass
        except Exception as e:
            logging.exception(e)
        
        jslink = JsLeaks().pool(self.js)
        
        self.result.extend(jslink)
        self.result = list(set(self.result))
        
        for i in self.result:
            console('Crawl', self.host, i + '\n')
        
        Sqldb(self.dbname).get_crawl(self.domain, self.result) 

# 需要导入模块: import concurrent [as 别名]
# 或者: from concurrent import futures [as 别名]
def test_future_traceback(self):
        @return_future
        @gen.engine
        def f(callback):
            yield gen.Task(self.io_loop.add_callback)
            try:
                1 / 0
            except ZeroDivisionError:
                self.expected_frame = traceback.extract_tb(
                    sys.exc_info()[2], limit=1)[0]
                raise
        try:
            yield f()
            self.fail("didn't get expected exception")
        except ZeroDivisionError:
            tb = traceback.extract_tb(sys.exc_info()[2])
            self.assertIn(self.expected_frame, tb)

# The following series of classes demonstrate and test various styles
# of use, with and without generators and futures. 

# 需要导入模块: import concurrent [as 别名]
# 或者: from concurrent import futures [as 别名]
def then(self, fn, deferred=False):
        result = Future(self._loop)
        result._register()

        def callback(_):
            try:
                if deferred:
                    temp = self._loop.run_later(fn, self.result())
                    temp.chain(result)
                elif not threading.current_thread() is self._loop:
                    temp = self._loop.run_async(fn, self.result())
                    temp.chain(result)
                else:
                    result.set_result(fn(self.result()))
            except Exception as e:
                self.logger.exception(
                    "Unhandled exception while chaining futures"
                )
                result.set_exception(e)
            except:
                result.cancel()

        self.add_done_callback(callback)
        return result 

# 需要导入模块: import concurrent [as 别名]
# 或者: from concurrent import futures [as 别名]
def run_async(self, func, *args, **kwds):
        """
        Fills in a list with the function to be executed in the pomp thread
        and wakes up the pomp thread.
        """
        future = Future(self)
        future._register()

        if threading.current_thread() is not self:
            self.async_pomp_task.append((future, func, args, kwds))
            self._wake_up()
        else:
            try:
                ret = func(*args, **kwds)
            except Exception as e:
                self.logger.exception(
                    "Unhandled exception in async task function"
                )
                future.set_exception(e)
            else:
                if not isinstance(ret, concurrent.futures.Future):
                    future.set_result(ret)
                else:
                    ret.chain(future)
        return future 

# 需要导入模块: import concurrent [as 别名]
# 或者: from concurrent import futures [as 别名]
def _run_task_list(self, task_list):
        """
        execute all pending functions located in the task list
        this is done in the order the list has been filled in
        """
        while len(task_list):
            future, f, args, kwds = task_list.pop(0)
            try:
                ret = f(*args, **kwds)
            except Exception as e:
                self.logger.exception(
                    "Unhandled exception in async task function"
                )
                self._unregister_future(future, ignore_error=True)
                future.set_exception(e)
                continue
            if not isinstance(ret, concurrent.futures.Future):
                future.set_result(ret)
            else:
                ret.chain(future) 

# 需要导入模块: import concurrent [as 别名]
# 或者: from concurrent import futures [as 别名]
def __init__(self, *args: Future, **kwargs: Future) -> None:
        if args and kwargs:
            raise ValueError("You must provide args or kwargs, not both")

        if kwargs:
            self._unfinished = dict((f, k) for (k, f) in kwargs.items())
            futures = list(kwargs.values())  # type: Sequence[Future]
        else:
            self._unfinished = dict((f, i) for (i, f) in enumerate(args))
            futures = args

        self._finished = collections.deque()  # type: Deque[Future]
        self.current_index = None  # type: Optional[Union[str, int]]
        self.current_future = None  # type: Optional[Future]
        self._running_future = None  # type: Optional[Future]

        for future in futures:
            future_add_done_callback(future, self._done_callback) 

# 需要导入模块: import concurrent [as 别名]
# 或者: from concurrent import futures [as 别名]
def download_file(self, bucket, key, filename, object_size,
                      extra_args, callback=None):
        with self._executor_cls(max_workers=2) as controller:
            # 1 thread for the future that manages the uploading of files
            # 1 thread for the future that manages IO writes.
            download_parts_handler = functools.partial(
                self._download_file_as_future,
                bucket, key, filename, object_size, callback)
            parts_future = controller.submit(download_parts_handler)

            io_writes_handler = functools.partial(
                self._perform_io_writes, filename)
            io_future = controller.submit(io_writes_handler)
            results = concurrent.futures.wait(
                [parts_future, io_future],
                return_when=concurrent.futures.FIRST_EXCEPTION)
            self._process_future_results(results) 

# 需要导入模块: import concurrent [as 别名]
# 或者: from concurrent import futures [as 别名]
def download_all_images(self, photo_list: [OSCPhoto],
                            track_path: str,
                            override=False,
                            workers: int = 10):
        """This method will download all images to a path overriding or not the files at
        that path. By default this method uses 10 parallel workers."""
        with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
            loop = asyncio.new_event_loop()
            futures = [
                loop.run_in_executor(executor,
                                     self.get_image, photo, track_path, override)
                for photo in photo_list
            ]
            if not futures:
                loop.close()
                return

            loop.run_until_complete(asyncio.gather(*futures))
            loop.close() 

# 需要导入模块: import concurrent [as 别名]
# 或者: from concurrent import futures [as 别名]
def execute_runners(self, parallel=False, num_threads=1):
        '''
        Run each of the algorithms
        '''

        base_output_dir = self.output_settings.base_dir

        batches =  self.runners.keys()

        for batch in batches:
            if parallel==True:
                executor = concurrent.futures.ThreadPoolExecutor(max_workers=1)
                futures = [executor.submit(runner.run, base_output_dir)
                    for runner in self.runners[batch]]
                
                # https://stackoverflow.com/questions/35711160/detect-failed-tasks-in-concurrent-futures
                # Re-raise exception if produced
                for future in concurrent.futures.as_completed(futures):
                    future.result()
                executor.shutdown(wait=True)
            else:
                for runner in self.runners[batch]:
                    runner.run(output_dir=base_output_dir) 

# 需要导入模块: import concurrent [as 别名]
# 或者: from concurrent import futures [as 别名]
def test_temp_table_concurrency(con, test_data_dir):
    # we don't install futures on windows in CI and we can't run this test
    # there anyway so we import here
    import concurrent.futures
    from concurrent.futures import as_completed

    def limit_10(i, hdfs_path):
        t = con.parquet_file(hdfs_path)
        return t.sort_by(t.r_regionkey).limit(1, offset=i).execute()

    nthreads = 4
    hdfs_path = pjoin(test_data_dir, 'parquet/tpch_region')

    with concurrent.futures.ThreadPoolExecutor(max_workers=nthreads) as e:
        futures = [e.submit(limit_10, i, hdfs_path) for i in range(nthreads)]
    assert all(map(len, (future.result() for future in as_completed(futures)))) 

# 需要导入模块: import concurrent [as 别名]
# 或者: from concurrent import futures [as 别名]
def test_async_await(self):
        class Object(object):
            def __init__(self):
                self.executor = futures.thread.ThreadPoolExecutor(1)

            @run_on_executor()
            def f(self):
                return 42

        o = Object()
        namespace = exec_test(globals(), locals(), """
        async def f():
            answer = await o.f()
            return answer
        """)
        result = yield namespace['f']()
        self.assertEqual(result, 42) 

# 需要导入模块: import concurrent [as 别名]
# 或者: from concurrent import futures [as 别名]
def wait_task(self, return_when=concurrent.futures.FIRST_COMPLETED):
        """Waits for one or more running tasks to complete.

        Results of all completed tasks are copied into ``self.ready_task_results``.

        When no task is running, does nothing.

        """
        if len(self.running_tasks) > 0:
            self.config.log("Waiting for tasks to complete...")
            ready_tasks, self.running_tasks = concurrent.futures.wait(
                self.running_tasks, return_when=return_when
            )
            for task in ready_tasks:
                self.ready_task_results.append(task.result())

    # Overridden such that instances of search job can be pickled to workers 

# 需要导入模块: import concurrent [as 别名]
# 或者: from concurrent import futures [as 别名]
def run(self, task):
        try:
            with concurrent.futures.ThreadPoolExecutor(max_workers=THREADS) as executor:
                futures = [executor.submit(self.scan, i) for i in task]
                for future in concurrent.futures.as_completed(futures, timeout=3):
                    future.result()

        except (EOFError, concurrent.futures._base.TimeoutError):
            pass

    # 创建启动任务 

# 需要导入模块: import concurrent [as 别名]
# 或者: from concurrent import futures [as 别名]
def test_no_calling(self):
        class Object(object):
            def __init__(self, io_loop):
                self.io_loop = io_loop
                self.executor = futures.thread.ThreadPoolExecutor(1)

            @run_on_executor
            def f(self):
                return 42

        o = Object(io_loop=self.io_loop)
        answer = yield o.f()
        self.assertEqual(answer, 42) 

# 需要导入模块: import concurrent [as 别名]
# 或者: from concurrent import futures [as 别名]
def test_call_with_no_args(self):
        class Object(object):
            def __init__(self, io_loop):
                self.io_loop = io_loop
                self.executor = futures.thread.ThreadPoolExecutor(1)

            @run_on_executor()
            def f(self):
                return 42

        o = Object(io_loop=self.io_loop)
        answer = yield o.f()
        self.assertEqual(answer, 42)