Python futures.wait方法代码示例

# 需要导入模块: from concurrent import futures [as 别名]
# 或者: from concurrent.futures import wait [as 别名]
def test_all_completed(self):
        future1 = self.executor.submit(divmod, 2, 0)
        future2 = self.executor.submit(mul, 2, 21)

        finished, pending = futures.wait(
                [SUCCESSFUL_FUTURE,
                 CANCELLED_AND_NOTIFIED_FUTURE,
                 EXCEPTION_FUTURE,
                 future1,
                 future2],
                return_when=futures.ALL_COMPLETED)

        self.assertEqual(set([SUCCESSFUL_FUTURE,
                              CANCELLED_AND_NOTIFIED_FUTURE,
                              EXCEPTION_FUTURE,
                              future1,
                              future2]), finished)
        self.assertEqual(set(), pending) 

# 需要导入模块: from concurrent import futures [as 别名]
# 或者: from concurrent.futures import wait [as 别名]
def await_future_or_eol(connection_observer, remain_time, start_time, timeout, logger):
    # Observer lifetime started with its timeout clock
    # but setting connection_observer._future may be delayed by nonempty commands queue.
    # In such case we have to wait either for _future or timeout.
    end_of_life = False
    while (connection_observer._future is None) and (remain_time > 0.0):
        time.sleep(0.005)
        if connection_observer.done():
            logger.debug("{} is done before creating future".format(connection_observer))
            end_of_life = True
            break
        now = time.time()
        already_passed = now - start_time
        remain_time = timeout - already_passed
        observer_lifetime_passed = now - connection_observer.life_status.start_time
        remain_observer_lifetime = connection_observer.timeout + connection_observer.life_status.terminating_timeout\
            - observer_lifetime_passed
        # we timeout on earlier timeout (timeout or connection_observer.timeout)
        if remain_observer_lifetime <= 0.0:
            remain_time = 0.0
        if remain_time <= 0.0:
            logger.debug("{} timeout before creating future".format(connection_observer))

    return end_of_life, remain_time 

# 需要导入模块: from concurrent import futures [as 别名]
# 或者: from concurrent.futures import wait [as 别名]
def test_infinite_loop_stops(self):
        """An infinite loop can be stopped after at least one iteration."""

        class Inc(Runnable):
            def __init__(self):
                super(Inc, self).__init__()
                self.first_run = threading.Event()

            def next(self, state):
                self.first_run.set()
                return state.updated(cnt=state.cnt + 1)

        loop = Loop(Inc())
        state = loop.run(State(cnt=0))

        # make sure loop body runnable is run at least once, then issue stop
        loop.runnable.first_run.wait(timeout=1)
        loop.stop()

        self.assertTrue(state.result().cnt >= 1) 

# 需要导入模块: from concurrent import futures [as 别名]
# 或者: from concurrent.futures import wait [as 别名]
def test_basic(self):
        wait = Wait()
        state = State(x=1)
        out = wait.run(state)

        # wait block should not finish
        done, not_done = futures.wait({out}, timeout=0.1)
        self.assertEqual(len(done), 0)
        self.assertEqual(len(not_done), 1)

        # until we stop it
        wait.stop()

        done, not_done = futures.wait({out}, timeout=0.1)
        self.assertEqual(len(done), 1)
        self.assertEqual(len(not_done), 0)

        self.assertEqual(out.result().x, 1) 

# 需要导入模块: from concurrent import futures [as 别名]
# 或者: from concurrent.futures import wait [as 别名]
def perform_requests(self):
        signal.signal(signal.SIGINT, self.exit_fast)
        signal.signal(signal.SIGTERM, self.exit_fast)

        self.state = b'E'
        for q_batch in self.get_batch():
            for (_, _) in self.split_batch(q_batch):
                if self.state != b"R":
                    self.state = b'R'
                yield
                continue

        #  wait for all batches to finish before returning
        self.state = b'W'
        while self.futures:
            f_len = len(self.futures)
            self.futures = [i for i in self.futures if not i.done()]
            if f_len != len(self.futures):
                self.ui.debug('Waiting for final requests to finish. '
                              'remaining requests: {}'
                              ''.format(len(self.futures)))
            wait(self.futures, return_when=FIRST_COMPLETED)
        self.state = b'D'
        yield True 

# 需要导入模块: from concurrent import futures [as 别名]
# 或者: from concurrent.futures import wait [as 别名]
def test_timeout(self):
        future1 = self.executor.submit(mul, 6, 7)
        future2 = self.executor.submit(time.sleep, 6)

        finished, pending = futures.wait(
                [CANCELLED_AND_NOTIFIED_FUTURE,
                 EXCEPTION_FUTURE,
                 SUCCESSFUL_FUTURE,
                 future1, future2],
                timeout=5,
                return_when=futures.ALL_COMPLETED)

        self.assertEqual(set([CANCELLED_AND_NOTIFIED_FUTURE,
                              EXCEPTION_FUTURE,
                              SUCCESSFUL_FUTURE,
                              future1]), finished)
        self.assertEqual(set([future2]), pending) 

# 需要导入模块: from concurrent import futures [as 别名]
# 或者: from concurrent.futures import wait [as 别名]
def _block_and_handle_missing(method):
    """For ModelFuture file-retrieving property methods.
    Block until completion and attempt to retrieve result.
    Raise exception only if the result isn't found.
    """
    @wraps(method)
    def wrapper(self):
        futures.wait((self,))  # Block until done
        try:
            return method(self)
        except FileNotFoundError:
            # We get here if the modeling job failed to produce
            # any output and we don't have metadata.
            if self.exception():
                raise self.exception() from None
            else:
                raise
    return wrapper 

# 需要导入模块: from concurrent import futures [as 别名]
# 或者: from concurrent.futures import wait [as 别名]
def _wait_for(fs, no_green_return_when, on_all_green_cb,
              caller_name, timeout=None):
    green_fs = sum(1 for f in fs if isinstance(f, futurist.GreenFuture))
    if not green_fs:
        done, not_done = futures.wait(fs, timeout=timeout,
                                      return_when=no_green_return_when)
        return DoneAndNotDoneFutures(done, not_done)
    else:
        non_green_fs = len(fs) - green_fs
        if non_green_fs:
            raise RuntimeError("Can not wait on %s green futures and %s"
                               " non-green futures in the same"
                               " `%s` call" % (green_fs, non_green_fs,
                                               caller_name))
        else:
            return on_all_green_cb(fs, timeout=timeout) 

# 需要导入模块: from concurrent import futures [as 别名]
# 或者: from concurrent.futures import wait [as 别名]
def _wait_for_all_green(fs, timeout=None):
    if not fs:
        return DoneAndNotDoneFutures(set(), set())

    with _acquire_and_release_futures(fs):
        done, not_done = _partition_futures(fs)
        if len(done) == len(fs):
            return DoneAndNotDoneFutures(done, not_done)
        waiter = _create_and_install_waiters(not_done,
                                             _AllGreenWaiter,
                                             len(not_done))
    waiter.event.wait(timeout)
    for f in not_done:
        with f._condition:
            f._waiters.remove(waiter)

    with _acquire_and_release_futures(fs):
        done, not_done = _partition_futures(fs)
        return DoneAndNotDoneFutures(done, not_done) 

# 需要导入模块: from concurrent import futures [as 别名]
# 或者: from concurrent.futures import wait [as 别名]
def _wait_for_any_green(fs, timeout=None):
    if not fs:
        return DoneAndNotDoneFutures(set(), set())

    with _acquire_and_release_futures(fs):
        done, not_done = _partition_futures(fs)
        if done:
            return DoneAndNotDoneFutures(done, not_done)
        waiter = _create_and_install_waiters(fs, _AnyGreenWaiter)

    waiter.event.wait(timeout)
    for f in fs:
        with f._condition:
            f._waiters.remove(waiter)

    with _acquire_and_release_futures(fs):
        done, not_done = _partition_futures(fs)
        return DoneAndNotDoneFutures(done, not_done) 

# 需要导入模块: from concurrent import futures [as 别名]
# 或者: from concurrent.futures import wait [as 别名]
def get_latch(num):
    """Get a countdown latch function for use in n threads."""
    cv = threading.Condition()
    count = 0

    def countdown_latch():
        """Block until n-1 other threads have called."""
        nonlocal count
        cv.acquire()
        count += 1
        cv.notify()
        cv.release()
        cv.acquire()
        while count < num:
            cv.wait()
        cv.release()

    return countdown_latch 

# 需要导入模块: from concurrent import futures [as 别名]
# 或者: from concurrent.futures import wait [as 别名]
def process(self, queue, workflow):
        while queue.__futures__:
            done, _ = wait(queue.__futures__, return_when=FIRST_COMPLETED)
            queue.progress(done)
        return workflow.result() 

# 需要导入模块: from concurrent import futures [as 别名]
# 或者: from concurrent.futures import wait [as 别名]
def download_batch(self, batch: Mapping[str, DatabaseMedia]):
        """ Downloads a batch of media items collected in download_photo_media.

        A fresh 'base_url' is required since they have limited lifespan and
        these are obtained by a single call to the service function
        mediaItems.batchGet.
        """
        try:
            response = self._api.mediaItems.batchGet.execute(mediaItemIds=batch.keys())
            r_json = response.json()
            if r_json.get("pageToken"):
                log.error("Ops - Batch size too big, some items dropped!")

            for i, result in enumerate(r_json["mediaItemResults"]):
                media_item_json = result.get("mediaItem")
                if not media_item_json:
                    log.warning("Null response in mediaItems.batchGet %s", batch.keys())
                    log.debug(
                        "Null response in mediaItems.batchGet"
                        "for item %d in\n\n %s \n\n which is \n%s",
                        i,
                        str(r_json),
                        str(result),
                    )
                else:
                    media_item = batch.get(media_item_json["id"])
                    self.download_file(media_item, media_item_json)
        except RequestException:
            self.find_bad_items(batch)

        except KeyboardInterrupt:
            log.warning("Cancelling download threads ...")
            for f in self.pool_future_to_media:
                f.cancel()
            futures.wait(self.pool_future_to_media)
            log.warning("Cancelled download threads")
            raise 

# 需要导入模块: from concurrent import futures [as 别名]
# 或者: from concurrent.futures import wait [as 别名]
def download_file(self, media_item: DatabaseMedia, media_json: dict):
        """ farms a single media download off to the thread pool.

        Uses a dictionary of Futures -> mediaItem to track downloads that are
        currently scheduled/running. When a Future is done it calls
        do_download_complete to remove the Future from the dictionary and
        complete processing of the media item.
        """
        base_url = media_json["baseUrl"]

        # we dont want a massive queue so wait until at least one thread is free
        while len(self.pool_future_to_media) >= self.max_threads:
            # check which futures are done, complete the main thread work
            # and remove them from the dictionary
            done_list = []
            for future in self.pool_future_to_media.keys():
                if future.done():
                    done_list.append(future)

            self.do_download_complete(done_list)

        # start a new background download
        self.files_download_started += 1
        log.info(
            "downloading %d %s", self.files_download_started, media_item.relative_path
        )
        future = self.download_pool.submit(self.do_download_file, base_url, media_item)
        self.pool_future_to_media[future] = media_item 

# 需要导入模块: from concurrent import futures [as 别名]
# 或者: from concurrent.futures import wait [as 别名]
def _shutdown_thread_pools(self) -> None:
        self._queued_server_scans = []
        for thread_pool in self._thread_pools:
            thread_pool.shutdown(wait=True)
        self._thread_pools = []

        # Force garbage collection because for some reason the Future objects created by ThreadPoolExecutor.submit()
        # take a ton of memory (compared to what they do - holding a function to call and its arguments):
        # https://stackoverflow.com/questions/45946274/rss-memory-usage-from-concurrent-futures
        # https://stackoverflow.com/questions/53104082/using-threadpoolexecutor-with-reduced-memory-footprint
        # https://stackoverflow.com/questions/34770169/using-concurrent-futures-without-running-out-of-ram
        # We force garbage collection here to ensure memory usage does not balloon when running SSLyze in some kind
        # of long-running app (such as a web app). Otherwise, the GC tends to not cleanup all the Future objects right
        # away (although at this point, all the work has been completed) and memory usage goes up like crazy
        gc.collect()