Python asyncio.as_completed方法代码示例

# 需要导入模块: import asyncio [as 别名]
# 或者: from asyncio import as_completed [as 别名]
def clean_daily_bar():
    day = datetime.datetime.strptime('20100416', '%Y%m%d').replace(tzinfo=pytz.FixedOffset(480))
    end = datetime.datetime.strptime('20160118', '%Y%m%d').replace(tzinfo=pytz.FixedOffset(480))
    tasks = []
    while day <= end:
        tasks.append(is_trading_day(day))
        day += datetime.timedelta(days=1)
    trading_days = []
    for f in tqdm(asyncio.as_completed(tasks), total=len(tasks)):
        rst = await f
        trading_days.append(rst)
    tasks.clear()
    for day, trading in trading_days:
        if not trading:
            DailyBar.objects.filter(time=day.date()).delete()
    print('done!') 

# 需要导入模块: import asyncio [as 别名]
# 或者: from asyncio import as_completed [as 别名]
def run(self, host):
        tasks = []
        # 默认limit=100，enable_cleanup_closed设置为True防止ssl泄露，ttl_dns_cache调高dns缓存
        conn = aiohttp.TCPConnector(
            limit=LIMIT,
            enable_cleanup_closed=True,
            ttl_dns_cache=100,
            ssl=False,
        )
        timeout = aiohttp.ClientTimeout(total=60, connect=2)
        async with aiohttp.ClientSession(connector=conn, timeout=timeout) as session:
            for url in self.urls:
                task = asyncio.ensure_future(self.scan(host, url, session))
                tasks.append(task)
            # gather方法是所有请求完成后才有输出
            _ = await asyncio.gather(*tasks)
            # for i in asyncio.as_completed(tasks):  # 类似于线程池中的task一样
            #     answer = await i
    
    # 创建启动任务 

# 需要导入模块: import asyncio [as 别名]
# 或者: from asyncio import as_completed [as 别名]
def fetch_bar():
    day = datetime.datetime.strptime('20100416', '%Y%m%d').replace(tzinfo=pytz.FixedOffset(480))
    end = datetime.datetime.strptime('20160118', '%Y%m%d').replace(tzinfo=pytz.FixedOffset(480))
    tasks = []
    while day <= end:
        tasks.append(is_trading_day(day))
        day += datetime.timedelta(days=1)
    trading_days = []
    for f in tqdm(asyncio.as_completed(tasks), total=len(tasks)):
        rst = await f
        trading_days.append(rst)
    tasks.clear()
    for day, trading in trading_days:
        if trading:
            tasks += [
                asyncio.ensure_future(update_from_shfe(day)),
                asyncio.ensure_future(update_from_dce(day)),
                asyncio.ensure_future(update_from_czce(day)),
                asyncio.ensure_future(update_from_cffex(day)),
            ]
    print('task len=', len(tasks))
    for f in tqdm(asyncio.as_completed(tasks), total=len(tasks)):
        await f 

# 需要导入模块: import asyncio [as 别名]
# 或者: from asyncio import as_completed [as 别名]
def main():
    # Instantiate a Meraki dashboard API session
    # NOTE: you have to use "async with" so that the session will be closed correctly at the end of the usage
    async with meraki.aio.AsyncDashboardAPI(
            api_key,
            base_url="https://api.meraki.com/api/v0",
            log_file_prefix=__file__[:-3],
            print_console=False,
    ) as aiomeraki:
        # Get list of organizations to which API key has access
        organizations = await aiomeraki.organizations.getOrganizations()

        # create a list of all organizations so we can call them all concurrently
        organizationTasks = [listOrganization(aiomeraki, org) for org in organizations]
        for task in asyncio.as_completed(organizationTasks):
            # as_completed returns an iterator, so we just have to await the iterator and not call it
            organizationName = await task
            print(f"finished organization: {organizationName}")

        print("Script complete!") 

# 需要导入模块: import asyncio [as 别名]
# 或者: from asyncio import as_completed [as 别名]
def main():
    # Instantiate a Meraki dashboard API session
    # NOTE: you have to use "async with" so that the session will be closed correctly at the end of the usage
    async with meraki.aio.AsyncDashboardAPI(
            api_key,
            base_url="https://api.meraki.com/api/v1",
            log_file_prefix=__file__[:-3],
            print_console=False,
    ) as aiomeraki:
        # Get list of organizations to which API key has access
        organizations = await aiomeraki.organizations.getOrganizations()

        # create a list of all organizations so we can call them all concurrently
        organizationTasks = [listOrganization(aiomeraki, org) for org in organizations]
        for task in asyncio.as_completed(organizationTasks):
            # as_completed returns an iterator, so we just have to await the iterator and not call it
            organizationName = await task
            print(f"finished organization: {organizationName}")

        print("Script complete!") 

# 需要导入模块: import asyncio [as 别名]
# 或者: from asyncio import as_completed [as 别名]
def async_fetch(cls, urls, descs=None, cb=None, datas=None, fds=None):
        if descs is None:
            descs = []
        if datas is None:
            datas = []
        if fds is None:
            fds = []
        conn = aiohttp.TCPConnector(limit_per_host=cls.CONNECTIONS_PER_HOST)
        async with aiohttp.ClientSession(
                connector=conn,
                headers={'User-Agent': cls.USER_AGENT}
        ) as session:
            coros = [
                asyncio.ensure_future(cls._async_fetch_one(session, url, desc, cb, data, fd))
                for url, desc, data, fd in zip_longest(urls, descs, datas, fds)
            ]
            with tqdm(asyncio.as_completed(coros),
                      total=len(coros),
                      desc="Downloading", unit="files") as t:
                result = [await coro for coro in t]
        return result 

# 需要导入模块: import asyncio [as 别名]
# 或者: from asyncio import as_completed [as 别名]
def wait(self, timeout: float = None) -> None:
        if self.triggered_token is not None:
            return

        futures = [asyncio.ensure_future(self._triggered.wait(), loop=self.loop)]
        for token in self._chain:
            futures.append(asyncio.ensure_future(token.wait(), loop=self.loop))

        if timeout is not None:
            futures.append(asyncio.ensure_future(asyncio.sleep(timeout), loop=self.loop))

        def cancel_not_done(fut: 'asyncio.Future[None]') -> None:
            for future in futures:
                if not future.done():
                    future.cancel()

        async def _wait_for_first(futures: Sequence[Awaitable[Any]]) -> None:
            for future in asyncio.as_completed(futures):
                await cast(Awaitable[Any], future)
                return

        fut = asyncio.ensure_future(_wait_for_first(futures), loop=self.loop)
        fut.add_done_callback(cancel_not_done)
        await fut 

# 需要导入模块: import asyncio [as 别名]
# 或者: from asyncio import as_completed [as 别名]
def test_as_completed_with_unused_timeout(self):

        def gen():
            yield
            yield 0
            yield 0.01

        loop = self.new_test_loop(gen)

        a = asyncio.sleep(0.01, 'a', loop=loop)

        @asyncio.coroutine
        def foo():
            for f in asyncio.as_completed([a], timeout=1, loop=loop):
                v = yield from f
                self.assertEqual(v, 'a')

        loop.run_until_complete(asyncio.Task(foo(), loop=loop)) 

# 需要导入模块: import asyncio [as 别名]
# 或者: from asyncio import as_completed [as 别名]
def test_as_completed_reverse_wait(self):

        def gen():
            yield 0
            yield 0.05
            yield 0

        loop = self.new_test_loop(gen)

        a = asyncio.sleep(0.05, 'a', loop=loop)
        b = asyncio.sleep(0.10, 'b', loop=loop)
        fs = {a, b}
        futs = list(asyncio.as_completed(fs, loop=loop))
        self.assertEqual(len(futs), 2)

        x = loop.run_until_complete(futs[1])
        self.assertEqual(x, 'a')
        self.assertAlmostEqual(0.05, loop.time())
        loop.advance_time(0.05)
        y = loop.run_until_complete(futs[0])
        self.assertEqual(y, 'b')
        self.assertAlmostEqual(0.10, loop.time()) 

# 需要导入模块: import asyncio [as 别名]
# 或者: from asyncio import as_completed [as 别名]
def test_as_completed_concurrent(self):

        def gen():
            when = yield
            self.assertAlmostEqual(0.05, when)
            when = yield 0
            self.assertAlmostEqual(0.05, when)
            yield 0.05

        loop = self.new_test_loop(gen)

        a = asyncio.sleep(0.05, 'a', loop=loop)
        b = asyncio.sleep(0.05, 'b', loop=loop)
        fs = {a, b}
        futs = list(asyncio.as_completed(fs, loop=loop))
        self.assertEqual(len(futs), 2)
        waiter = asyncio.wait(futs, loop=loop)
        done, pending = loop.run_until_complete(waiter)
        self.assertEqual(set(f.result() for f in done), {'a', 'b'}) 

# 需要导入模块: import asyncio [as 别名]
# 或者: from asyncio import as_completed [as 别名]
def main():
    coroutine1 = do_some_work(1)
    coroutine2 = do_some_work(2)
    coroutine3 = do_some_work(4)
    tasks = [
        asyncio.ensure_future(coroutine1),
        asyncio.ensure_future(coroutine2),
        asyncio.ensure_future(coroutine3)
    ]
    return await asyncio.gather(*tasks)
    # # 这里使用await等待另一个或多个协程运行完
    # dones, pendings = await asyncio.wait(tasks)
    # for task in dones:
    #     print("Task ret:", task.result())

    # results = await asyncio.gather(*tasks)
    # for result in results:
    #     print("Task ret:",result)

    # for task in asyncio.as_completed(tasks):
    #     result = await task
    #     print("Task ret: {}".format(result)) 

# 需要导入模块: import asyncio [as 别名]
# 或者: from asyncio import as_completed [as 别名]
def make_calls(dashboard, calls):
    global COMPLETED_OPERATIONS, DEVICES, NETWORKS, TEMPLATES

    tasks = [async_call(dashboard, call) for call in calls]
    for task in asyncio.as_completed(tasks):
        results = await task
        if results:
            operation = results['operation']
            response = results['response']
            file_name = results['file_name']
            file_path = results['file_path']

            save_data(file_name, response, file_path)

            # Update global variables
            COMPLETED_OPERATIONS.add(operation)
            if operation == 'getOrganizationNetworks':
                NETWORKS = response
            elif operation == 'getOrganizationConfigTemplates':
                TEMPLATES = response
            elif operation == 'getOrganizationDevices':
                DEVICES = response


# Backup configuration for organization 

# 需要导入模块: import asyncio [as 别名]
# 或者: from asyncio import as_completed [as 别名]
def get_picture_urls(dates, verbose=False):
    semaphore = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)
    tasks = [get_picture_url(date, semaphore) for date in dates]
    urls = []
    count = 0
    # get results as jobs are done
    for job in asyncio.as_completed(tasks, timeout=GLOBAL_TIMEOUT):
        try:
            url = yield from job
        except NoPictureForDate as exc:
            if verbose:
                print('*** {!r} ***'.format(exc))
            continue
        except aiohttp.ClientResponseError as exc:
            print('****** {!r} ******'.format(exc))
            continue
        count += 1
        if verbose:
            print(format(count, '3d'), end=' ')
            print(url.split('/')[-1])
        else:
            print(url)
        urls.append(url)
    return urls 

# 需要导入模块: import asyncio [as 别名]
# 或者: from asyncio import as_completed [as 别名]
def run(self,queue):
        self.start_time = datetime.now()
        for _ in range(self.NOT_FOUND_ATTEMPTS):
            for ext in self.extensions:
                uri = (random_nstring(20)+ext).strip()
                success = yield from self.not_found_probe(uri)
                if not success:
                    self.terminalw.print_error("404 detection failed")
                    self.save_output()
                    return -1
        count = 0
        total = len(queue)
        for subset in grouper(10000,queue):
            if subset:
                coros = []
                for x in subset:
                    if x:
                        coros.append(asyncio.Task(self.probe(x),loop=self.loop))

                for f in self.pb.tqdm(asyncio.as_completed(coros),start=count,total=total,desc=self.host.geturl(),miniters=10):
                    yield from f
                count += len(coros)
        self.save_output()
        self.end() 

# 需要导入模块: import asyncio [as 别名]
# 或者: from asyncio import as_completed [as 别名]
def watcher(tasks,delay=False):
    res = []
    for t in asyncio.as_completed(tasks):
        r = yield from t
        res.append(r)
        if delay:
            # simulate processing delay
            process_time = random.random() / 10
            yield from asyncio.sleep(process_time)
    #print(res)
    #assert(sorted(res) == res)
    if sorted(res) != res:
        print('FAIL', res)
        print('------------')
    else:
        print('.', end='')
        sys.stdout.flush()