本文整理匯總了Python中types.AsyncGeneratorType方法的典型用法代碼示例。如果您正苦於以下問題:Python types.AsyncGeneratorType方法的具體用法?Python types.AsyncGeneratorType怎麽用?Python types.AsyncGeneratorType使用的例子?那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類types
的用法示例。
在下文中一共展示了types.AsyncGeneratorType方法的12個代碼示例,這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚,您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: handle_request
# 需要導入模塊: import types [as 別名]
# 或者: from types import AsyncGeneratorType [as 別名]
def handle_request(
self, request: Request
) -> typing.Tuple[AsyncGeneratorType, Response]:
"""
Wrap request with middleware.
:param request:
:return:
"""
callback_result, response = None, None
try:
await self._run_request_middleware(request)
callback_result, response = await request.fetch_callback(self.sem)
await self._run_response_middleware(request, response)
await self._process_response(request=request, response=response)
except NotImplementedParseError as e:
self.logger.error(e)
except NothingMatchedError as e:
error_info = f"<Field: {str(e).lower()}" + f", error url: {request.url}>"
self.logger.error(error_info)
except Exception as e:
self.logger.error(f"<Callback[{request.callback.__name__}]: {e}")
return callback_result, response
示例2: start_worker
# 需要導入模塊: import types [as 別名]
# 或者: from types import AsyncGeneratorType [as 別名]
def start_worker(self):
"""
Stark spider worker
:return:
"""
while True:
request_item = await self.request_queue.get()
self.worker_tasks.append(request_item)
if self.request_queue.empty():
results = await asyncio.gather(
*self.worker_tasks, return_exceptions=True
)
for task_result in results:
if not isinstance(task_result, RuntimeError) and task_result:
callback_results, response = task_result
if isinstance(callback_results, AsyncGeneratorType):
await self._process_async_callback(
callback_results, response
)
self.worker_tasks = []
self.request_queue.task_done()
示例3: fetch_callback
# 需要導入模塊: import types [as 別名]
# 或者: from types import AsyncGeneratorType [as 別名]
def fetch_callback(
self, sem: Semaphore
) -> Tuple[AsyncGeneratorType, Response]:
"""
Request the target url and then call the callback function
:param sem: Semaphore
:return: Tuple[AsyncGeneratorType, Response]
"""
try:
async with sem:
response = await self.fetch()
except Exception as e:
response = None
self.logger.error(f"<Error: {self.url} {e}>")
if self.callback is not None:
if iscoroutinefunction(self.callback):
callback_result = await self.callback(response)
else:
callback_result = self.callback(response)
else:
callback_result = None
return callback_result, response
示例4: as_future
# 需要導入模塊: import types [as 別名]
# 或者: from types import AsyncGeneratorType [as 別名]
def as_future(self, fun, *args, **kwargs):
try:
res = fun(*args, **kwargs)
except Exception:
return create_future_error(create_failure())
else:
if isinstance(res, Future):
return res
elif iscoroutine(res):
return self._loop.create_task(res)
elif isinstance(res, AsyncGeneratorType):
raise RuntimeError(
"as_future() received an async generator function; does "
"'{}' use 'yield' when you meant 'await'?".format(
str(fun)
)
)
else:
return create_future_success(res)
示例5: is_internal_attribute
# 需要導入模塊: import types [as 別名]
# 或者: from types import AsyncGeneratorType [as 別名]
def is_internal_attribute(obj, attr):
"""Test if the attribute given is an internal python attribute. For
example this function returns `True` for the `func_code` attribute of
python objects. This is useful if the environment method
:meth:`~SandboxedEnvironment.is_safe_attribute` is overridden.
>>> from jinja2.sandbox import is_internal_attribute
>>> is_internal_attribute(str, "mro")
True
>>> is_internal_attribute(str, "upper")
False
"""
if isinstance(obj, types.FunctionType):
if attr in UNSAFE_FUNCTION_ATTRIBUTES:
return True
elif isinstance(obj, types.MethodType):
if attr in UNSAFE_FUNCTION_ATTRIBUTES or \
attr in UNSAFE_METHOD_ATTRIBUTES:
return True
elif isinstance(obj, type):
if attr == 'mro':
return True
elif isinstance(obj, (types.CodeType, types.TracebackType, types.FrameType)):
return True
elif isinstance(obj, types.GeneratorType):
if attr in UNSAFE_GENERATOR_ATTRIBUTES:
return True
elif hasattr(types, 'CoroutineType') and isinstance(obj, types.CoroutineType):
if attr in UNSAFE_COROUTINE_ATTRIBUTES:
return True
elif hasattr(types, 'AsyncGeneratorType') and isinstance(obj, types.AsyncGeneratorType):
if attr in UNSAFE_ASYNC_GENERATOR_ATTRIBUTES:
return True
return attr.startswith('__')
示例6: _process_async_callback
# 需要導入模塊: import types [as 別名]
# 或者: from types import AsyncGeneratorType [as 別名]
def _process_async_callback(
self, callback_results: AsyncGeneratorType, response: Response = None
):
try:
async for callback_result in callback_results:
if isinstance(callback_result, AsyncGeneratorType):
await self._process_async_callback(callback_result)
elif isinstance(callback_result, Request):
self.request_queue.put_nowait(
self.handle_request(request=callback_result)
)
elif isinstance(callback_result, typing.Coroutine):
self.request_queue.put_nowait(
self.handle_callback(
aws_callback=callback_result, response=response
)
)
elif isinstance(callback_result, Item):
# Process target item
await self.process_item(callback_result)
else:
await self.process_callback_result(callback_result=callback_result)
except NothingMatchedError as e:
error_info = f"<Field: {str(e).lower()}" + f", error url: {response.url}>"
self.logger.error(error_info)
except Exception as e:
self.logger.error(e)
示例7: is_internal_attribute
# 需要導入模塊: import types [as 別名]
# 或者: from types import AsyncGeneratorType [as 別名]
def is_internal_attribute(obj, attr):
"""Test if the attribute given is an internal python attribute. For
example this function returns `True` for the `func_code` attribute of
python objects. This is useful if the environment method
:meth:`~SandboxedEnvironment.is_safe_attribute` is overridden.
>>> from jinja2.sandbox import is_internal_attribute
>>> is_internal_attribute(str, "mro")
True
>>> is_internal_attribute(str, "upper")
False
"""
if isinstance(obj, types.FunctionType):
if attr in UNSAFE_FUNCTION_ATTRIBUTES:
return True
elif isinstance(obj, types.MethodType):
if attr in UNSAFE_FUNCTION_ATTRIBUTES or attr in UNSAFE_METHOD_ATTRIBUTES:
return True
elif isinstance(obj, type):
if attr == "mro":
return True
elif isinstance(obj, (types.CodeType, types.TracebackType, types.FrameType)):
return True
elif isinstance(obj, types.GeneratorType):
if attr in UNSAFE_GENERATOR_ATTRIBUTES:
return True
elif hasattr(types, "CoroutineType") and isinstance(obj, types.CoroutineType):
if attr in UNSAFE_COROUTINE_ATTRIBUTES:
return True
elif hasattr(types, "AsyncGeneratorType") and isinstance(
obj, types.AsyncGeneratorType
):
if attr in UNSAFE_ASYNC_GENERATOR_ATTRIBUTES:
return True
return attr.startswith("__")
示例8: isasyncgen
# 需要導入模塊: import types [as 別名]
# 或者: from types import AsyncGeneratorType [as 別名]
def isasyncgen(object):
"""Return true if the object is an asynchronous generator."""
return isinstance(object, types.AsyncGeneratorType)
示例9: force_through
# 需要導入模塊: import types [as 別名]
# 或者: from types import AsyncGeneratorType [as 別名]
def force_through(__call__, self, x, through):
"""Clone an abstract value (asynchronous)."""
if not isinstance(x, through) and not isinstance(x, Pending):
return x
cache = self.state
if isinstance(x, AbstractValue) and x in cache:
return cache[x]
call = __call__(self, x, through)
if isinstance(call, AsyncGeneratorType):
cls = await call.asend(None)
inst = cls.empty()
cache[x] = inst
constructor = _make_constructor(inst)
rval = await call.asend(constructor)
assert rval is inst
return rval
else:
return await call
# Uncomment and test the other implementations if/when needed:
# @overload # noqa: F811
# async def force_through(self, x: AbstractScalar, through):
# return AbstractScalar(await self(x.values, through))
# @overload # noqa: F811
# async def force_through(self, x: AbstractFunction, through):
# yield (yield AbstractFunction)(*(await self(x.get_sync(), through)))
# @overload # noqa: F811
# async def force_through(self, d: TrackDict, through):
# return {k: await self(v, through) for k, v in d.items()}
示例10: gen
# 需要導入模塊: import types [as 別名]
# 或者: from types import AsyncGeneratorType [as 別名]
def gen(hub, payload, reader, writer):
'''
Run a generator and yield back the returns. Supports a generator and an
async generator
'''
ref = payload.get('ref')
args = payload.get('args', [])
kwargs = payload.get('kwargs', {})
ret = hub.pop.ref.last(ref)(*args, **kwargs)
if isinstance(ret, types.AsyncGeneratorType):
async for chunk in ret:
rchunk = msgpack.dumps(chunk, use_bin_type=True)
rchunk += hub.proc.I_FLAG
rchunk += hub.proc.DELIM
writer.write(rchunk)
await writer.drain()
elif isinstance(ret, types.GeneratorType):
for chunk in ret:
rchunk = msgpack.dumps(chunk, use_bin_type=True)
rchunk += hub.proc.I_FLAG
rchunk += hub.proc.DELIM
writer.write(rchunk)
await writer.drain()
elif asyncio.iscoroutine(ret):
return await ret
else:
return ret
return ''
示例11: addFeedNodes
# 需要導入模塊: import types [as 別名]
# 或者: from types import AsyncGeneratorType [as 別名]
def addFeedNodes(self, name, items):
'''
Call a feed function and return what it returns (typically yields Node()s).
Args:
name (str): The name of the feed record type.
items (list): A list of records of the given feed type.
Returns:
(object): The return value from the feed function. Typically Node() generator.
'''
func = self.core.getFeedFunc(name)
if func is None:
raise s_exc.NoSuchName(name=name)
logger.info(f'adding feed nodes ({name}): {len(items)}')
genr = func(self, items)
if not isinstance(genr, types.AsyncGeneratorType):
if isinstance(genr, types.CoroutineType):
genr.close()
mesg = f'feed func returned a {type(genr)}, not an async generator.'
raise s_exc.BadCtorType(mesg=mesg, name=name)
async for node in genr:
yield node
示例12: addFeedData
# 需要導入模塊: import types [as 別名]
# 或者: from types import AsyncGeneratorType [as 別名]
def addFeedData(self, name, items):
func = self.core.getFeedFunc(name)
if func is None:
raise s_exc.NoSuchName(name=name)
logger.info(f'adding feed data ({name}): {len(items)}')
retn = func(self, items)
# If the feed function is an async generator, run it...
if isinstance(retn, types.AsyncGeneratorType):
retn = [x async for x in retn]
elif s_coro.iscoro(retn):
await retn