Python types.AsyncGeneratorType方法代码示例

# 需要导入模块: import types [as 别名]
# 或者: from types import AsyncGeneratorType [as 别名]
def handle_request(
        self, request: Request
    ) -> typing.Tuple[AsyncGeneratorType, Response]:
        """
        Wrap request with middleware.
        :param request:
        :return:
        """
        callback_result, response = None, None

        try:
            await self._run_request_middleware(request)
            callback_result, response = await request.fetch_callback(self.sem)
            await self._run_response_middleware(request, response)
            await self._process_response(request=request, response=response)
        except NotImplementedParseError as e:
            self.logger.error(e)
        except NothingMatchedError as e:
            error_info = f"<Field: {str(e).lower()}" + f", error url: {request.url}>"
            self.logger.error(error_info)
        except Exception as e:
            self.logger.error(f"<Callback[{request.callback.__name__}]: {e}")

        return callback_result, response 

# 需要导入模块: import types [as 别名]
# 或者: from types import AsyncGeneratorType [as 别名]
def start_worker(self):
        """
        Stark spider worker
        :return:
        """
        while True:
            request_item = await self.request_queue.get()
            self.worker_tasks.append(request_item)
            if self.request_queue.empty():
                results = await asyncio.gather(
                    *self.worker_tasks, return_exceptions=True
                )
                for task_result in results:
                    if not isinstance(task_result, RuntimeError) and task_result:
                        callback_results, response = task_result
                        if isinstance(callback_results, AsyncGeneratorType):
                            await self._process_async_callback(
                                callback_results, response
                            )
                self.worker_tasks = []
            self.request_queue.task_done() 

# 需要导入模块: import types [as 别名]
# 或者: from types import AsyncGeneratorType [as 别名]
def fetch_callback(
        self, sem: Semaphore
    ) -> Tuple[AsyncGeneratorType, Response]:
        """
        Request the target url and then call the callback function
        :param sem: Semaphore
        :return: Tuple[AsyncGeneratorType, Response]
        """
        try:
            async with sem:
                response = await self.fetch()
        except Exception as e:
            response = None
            self.logger.error(f"<Error: {self.url} {e}>")

        if self.callback is not None:
            if iscoroutinefunction(self.callback):
                callback_result = await self.callback(response)
            else:
                callback_result = self.callback(response)
        else:
            callback_result = None
        return callback_result, response 

# 需要导入模块: import types [as 别名]
# 或者: from types import AsyncGeneratorType [as 别名]
def as_future(self, fun, *args, **kwargs):
        try:
            res = fun(*args, **kwargs)
        except Exception:
            return create_future_error(create_failure())
        else:
            if isinstance(res, Future):
                return res
            elif iscoroutine(res):
                return self._loop.create_task(res)
            elif isinstance(res, AsyncGeneratorType):
                raise RuntimeError(
                    "as_future() received an async generator function; does "
                    "'{}' use 'yield' when you meant 'await'?".format(
                        str(fun)
                    )
                )
            else:
                return create_future_success(res) 

# 需要导入模块: import types [as 别名]
# 或者: from types import AsyncGeneratorType [as 别名]
def is_internal_attribute(obj, attr):
    """Test if the attribute given is an internal python attribute.  For
    example this function returns `True` for the `func_code` attribute of
    python objects.  This is useful if the environment method
    :meth:`~SandboxedEnvironment.is_safe_attribute` is overridden.

    >>> from jinja2.sandbox import is_internal_attribute
    >>> is_internal_attribute(str, "mro")
    True
    >>> is_internal_attribute(str, "upper")
    False
    """
    if isinstance(obj, types.FunctionType):
        if attr in UNSAFE_FUNCTION_ATTRIBUTES:
            return True
    elif isinstance(obj, types.MethodType):
        if attr in UNSAFE_FUNCTION_ATTRIBUTES or \
           attr in UNSAFE_METHOD_ATTRIBUTES:
            return True
    elif isinstance(obj, type):
        if attr == 'mro':
            return True
    elif isinstance(obj, (types.CodeType, types.TracebackType, types.FrameType)):
        return True
    elif isinstance(obj, types.GeneratorType):
        if attr in UNSAFE_GENERATOR_ATTRIBUTES:
            return True
    elif hasattr(types, 'CoroutineType') and isinstance(obj, types.CoroutineType):
        if attr in UNSAFE_COROUTINE_ATTRIBUTES:
            return True
    elif hasattr(types, 'AsyncGeneratorType') and isinstance(obj, types.AsyncGeneratorType):
        if attr in UNSAFE_ASYNC_GENERATOR_ATTRIBUTES:
            return True
    return attr.startswith('__') 

# 需要导入模块: import types [as 别名]
# 或者: from types import AsyncGeneratorType [as 别名]
def _process_async_callback(
        self, callback_results: AsyncGeneratorType, response: Response = None
    ):
        try:
            async for callback_result in callback_results:
                if isinstance(callback_result, AsyncGeneratorType):
                    await self._process_async_callback(callback_result)
                elif isinstance(callback_result, Request):
                    self.request_queue.put_nowait(
                        self.handle_request(request=callback_result)
                    )
                elif isinstance(callback_result, typing.Coroutine):
                    self.request_queue.put_nowait(
                        self.handle_callback(
                            aws_callback=callback_result, response=response
                        )
                    )
                elif isinstance(callback_result, Item):
                    # Process target item
                    await self.process_item(callback_result)
                else:
                    await self.process_callback_result(callback_result=callback_result)
        except NothingMatchedError as e:
            error_info = f"<Field: {str(e).lower()}" + f", error url: {response.url}>"
            self.logger.error(error_info)
        except Exception as e:
            self.logger.error(e) 

# 需要导入模块: import types [as 别名]
# 或者: from types import AsyncGeneratorType [as 别名]
def is_internal_attribute(obj, attr):
    """Test if the attribute given is an internal python attribute.  For
    example this function returns `True` for the `func_code` attribute of
    python objects.  This is useful if the environment method
    :meth:`~SandboxedEnvironment.is_safe_attribute` is overridden.

    >>> from jinja2.sandbox import is_internal_attribute
    >>> is_internal_attribute(str, "mro")
    True
    >>> is_internal_attribute(str, "upper")
    False
    """
    if isinstance(obj, types.FunctionType):
        if attr in UNSAFE_FUNCTION_ATTRIBUTES:
            return True
    elif isinstance(obj, types.MethodType):
        if attr in UNSAFE_FUNCTION_ATTRIBUTES or attr in UNSAFE_METHOD_ATTRIBUTES:
            return True
    elif isinstance(obj, type):
        if attr == "mro":
            return True
    elif isinstance(obj, (types.CodeType, types.TracebackType, types.FrameType)):
        return True
    elif isinstance(obj, types.GeneratorType):
        if attr in UNSAFE_GENERATOR_ATTRIBUTES:
            return True
    elif hasattr(types, "CoroutineType") and isinstance(obj, types.CoroutineType):
        if attr in UNSAFE_COROUTINE_ATTRIBUTES:
            return True
    elif hasattr(types, "AsyncGeneratorType") and isinstance(
        obj, types.AsyncGeneratorType
    ):
        if attr in UNSAFE_ASYNC_GENERATOR_ATTRIBUTES:
            return True
    return attr.startswith("__") 

# 需要导入模块: import types [as 别名]
# 或者: from types import AsyncGeneratorType [as 别名]
def isasyncgen(object):
    """Return true if the object is an asynchronous generator."""
    return isinstance(object, types.AsyncGeneratorType) 

# 需要导入模块: import types [as 别名]
# 或者: from types import AsyncGeneratorType [as 别名]
def force_through(__call__, self, x, through):
    """Clone an abstract value (asynchronous)."""
    if not isinstance(x, through) and not isinstance(x, Pending):
        return x
    cache = self.state
    if isinstance(x, AbstractValue) and x in cache:
        return cache[x]

    call = __call__(self, x, through)
    if isinstance(call, AsyncGeneratorType):
        cls = await call.asend(None)
        inst = cls.empty()
        cache[x] = inst
        constructor = _make_constructor(inst)
        rval = await call.asend(constructor)
        assert rval is inst
        return rval
    else:
        return await call


# Uncomment and test the other implementations if/when needed:


# @overload  # noqa: F811
# async def force_through(self, x: AbstractScalar, through):
#     return AbstractScalar(await self(x.values, through))


# @overload  # noqa: F811
# async def force_through(self, x: AbstractFunction, through):
#     yield (yield AbstractFunction)(*(await self(x.get_sync(), through)))


# @overload  # noqa: F811
# async def force_through(self, d: TrackDict, through):
#     return {k: await self(v, through) for k, v in d.items()} 

# 需要导入模块: import types [as 别名]
# 或者: from types import AsyncGeneratorType [as 别名]
def gen(hub, payload, reader, writer):
    '''
    Run a generator and yield back the returns. Supports a generator and an
    async generator
    '''
    ref = payload.get('ref')
    args = payload.get('args', [])
    kwargs = payload.get('kwargs', {})
    ret = hub.pop.ref.last(ref)(*args, **kwargs)
    if isinstance(ret, types.AsyncGeneratorType):
        async for chunk in ret:
            rchunk = msgpack.dumps(chunk, use_bin_type=True)
            rchunk += hub.proc.I_FLAG
            rchunk += hub.proc.DELIM
            writer.write(rchunk)
            await writer.drain()
    elif isinstance(ret, types.GeneratorType):
        for chunk in ret:
            rchunk = msgpack.dumps(chunk, use_bin_type=True)
            rchunk += hub.proc.I_FLAG
            rchunk += hub.proc.DELIM
            writer.write(rchunk)
            await writer.drain()
    elif asyncio.iscoroutine(ret):
        return await ret
    else:
        return ret
    return '' 

# 需要导入模块: import types [as 别名]
# 或者: from types import AsyncGeneratorType [as 别名]
def addFeedNodes(self, name, items):
        '''
        Call a feed function and return what it returns (typically yields Node()s).

        Args:
            name (str): The name of the feed record type.
            items (list): A list of records of the given feed type.

        Returns:
            (object): The return value from the feed function. Typically Node() generator.

        '''
        func = self.core.getFeedFunc(name)
        if func is None:
            raise s_exc.NoSuchName(name=name)

        logger.info(f'adding feed nodes ({name}): {len(items)}')

        genr = func(self, items)
        if not isinstance(genr, types.AsyncGeneratorType):
            if isinstance(genr, types.CoroutineType):
                genr.close()
            mesg = f'feed func returned a {type(genr)}, not an async generator.'
            raise s_exc.BadCtorType(mesg=mesg, name=name)

        async for node in genr:
            yield node 

# 需要导入模块: import types [as 别名]
# 或者: from types import AsyncGeneratorType [as 别名]
def addFeedData(self, name, items):

        func = self.core.getFeedFunc(name)
        if func is None:
            raise s_exc.NoSuchName(name=name)

        logger.info(f'adding feed data ({name}): {len(items)}')

        retn = func(self, items)

        # If the feed function is an async generator, run it...
        if isinstance(retn, types.AsyncGeneratorType):
            retn = [x async for x in retn]
        elif s_coro.iscoro(retn):
            await retn