Python requests.ReadTimeout方法代码示例

# 需要导入模块: import requests [as 别名]
# 或者: from requests import ReadTimeout [as 别名]
def test_watch_restart(self, m_get):
        path = '/test'
        data = [{'object': {'metadata': {'name': 'obj%s' % i,
                                         'resourceVersion': i}}}
                for i in range(3)]
        lines = [jsonutils.dump_as_bytes(i) for i in data]

        m_resp = mock.MagicMock()
        m_resp.ok = True
        m_resp.iter_lines.side_effect = [lines, requests.ReadTimeout, lines]
        m_get.return_value = m_resp

        self.assertEqual(data * 2,
                         list(itertools.islice(self.client.watch(path),
                                               len(data) * 2)))
        self.assertEqual(3, m_get.call_count)
        self.assertEqual(3, m_resp.close.call_count)
        m_get.assert_any_call(
            self.base_url + path, headers={}, stream=True,
            params={"watch": "true"}, cert=(None, None), verify=False,
            timeout=(30, 60))
        m_get.assert_any_call(
            self.base_url + path, headers={}, stream=True,
            params={"watch": "true", "resourceVersion": 2}, cert=(None, None),
            verify=False, timeout=(30, 60)) 

# 需要导入模块: import requests [as 别名]
# 或者: from requests import ReadTimeout [as 别名]
def soupify_url(url, timeout=2, encoding='utf-8', **kwargs):
    """Given a url returns a BeautifulSoup object"""
    try:
        r = requests.get(url, timeout=timeout, **kwargs)
    except ReadTimeout:
        logger.info("[soupify_url] Request for %s timed out.", url)
        raise
    except Exception as e:
        logger.error(f"Request for {url} could not be resolved", exc_info=True)
        raise ConnectionError(repr(e))


    r.raise_for_status()
    r.encoding = encoding
    if r.status_code == 200:
        return BeautifulSoup(r.text, 'lxml')
    else:
        raise ConnectionError(
            f'{url} returned error status %s - ', r.status_code, r.reason
        ) 

# 需要导入模块: import requests [as 别名]
# 或者: from requests import ReadTimeout [as 别名]
def test_read_timeout(mock_requests):
    url = "http://example.com"

    def callback(resp):
        raise requests.ReadTimeout()

    mock_requests.add_callback('GET', url, callback)

    match = url_re.search(url)
    assert match
    mck = MagicMock()
    logger = MagicMock()

    assert print_url_title(match=match, message=mck, logger=logger) is None

    logger.debug.assert_called_with('Read timeout reached for %r', url) 

# 需要导入模块: import requests [as 别名]
# 或者: from requests import ReadTimeout [as 别名]
def check(self, proxy):
        """
        测试代理，返回测试结果
        :param proxy: 代理
        :return: 测试结果
        """
        try:
            response = requests.get(settings.TEST_URL, proxies={
                'http': 'http://' + proxy,
                'https': 'https://' + proxy
            }, timeout=settings.TEST_TIMEOUT)
            logger.debug(f'Using {proxy} to test {settings.TEST_URL}...')
            if response.status_code == 200:
                return True
        except (ConnectionError, ReadTimeout):
            return False 

# 需要导入模块: import requests [as 别名]
# 或者: from requests import ReadTimeout [as 别名]
def __get_page_html(self, url):
        """
        Get parse html page from requesting url
        :param url: Requesting url
        :returns: Parsed html page
        """
        try:
            page_html = requests.get(url, headers=self.headers, cookies=self.cookies, timeout=SCAN_DEEP[SEARCH_LEVEL - 1])
            if page_html.status_code == 429:
                time.sleep(SCAN_DEEP[SEARCH_LEVEL - 1])
                self.__get_page_html(url)
            return page_html.text
        except requests.ConnectionError as e:
            error_print("[!] Error: There is '%s' problem in requesting html page." % str(e))
            exit()
        except requests.ReadTimeout:
            return '' 

# 需要导入模块: import requests [as 别名]
# 或者: from requests import ReadTimeout [as 别名]
def request(session=None, method="GET", url="", token=None, **kwargs):
    if not session:
        session = get_session()
    if token:
        session.headers = getattr(session, 'headers') or {}
        session.headers.update({'X-CSRF-Token': token})

    if BASE_URL not in url:
        url = f'{BASE_URL}{url}'

    try:
        return session.request(method=method, url=url, timeout=(15, 15), **kwargs)
    except (ConnectionError, ReadTimeout):
        print(INTERNET_DOWN_MSG)
        sys.exit(1) 

# 需要导入模块: import requests [as 别名]
# 或者: from requests import ReadTimeout [as 别名]
def do_request(self, method, url, data=None, json=True, headers=None, ignore_failure=False):
        loop = asyncio.get_event_loop()
        if not headers:
            headers = self._authentication_client.session.headers
        try:
            if data is None:
                data = {}
            params = {
                "method": method,
                "url": url,
                "data": data,
                "timeout": self._authentication_client.timeout,
                "headers": headers
            }
            try:
                response = await loop.run_in_executor(None, functools.partial(self._authentication_client.session.request, **params))
            except (requests.Timeout, requests.ConnectTimeout, requests.ReadTimeout):
                raise BackendTimeout()
            except requests.ConnectionError:
                raise NetworkError

            if not ignore_failure:
                self.handle_status_code(response.status_code)

            if json:
                return response.json()
            else:
                return response

        except Exception as e:
            raise e 

# 需要导入模块: import requests [as 别名]
# 或者: from requests import ReadTimeout [as 别名]
def wemp_spider(url, site):
    """
    抓取微信内容
    :param url:
    :param site:
    :return:
    """
    if is_crawled_url(url):
        return

    try:
        rsp = requests.get(url, timeout=10)

        if rsp.ok:
            try:
                if get_host_name(rsp.url) == 'mp.weixin.qq.com':
                    title, author, content = parse_weixin_page(rsp)
                elif 'ershicimi.com' in get_host_name(rsp.url):
                    title, author, content = parse_ershicimi_page(rsp)
                else:
                    logger.warning(f'公众号域名解析异常：`{rsp.url}')
                    return
            except:
                logger.info(f'公众号内容解析异常：`{rsp.url}')
                return

            article = Article(title=title, author=author, site=site, uindex=current_ts(),
                              content=content, src_url=url)
            article.save()

            mark_crawled_url(url)
    except (ConnectTimeout, HTTPError, ReadTimeout, Timeout, ConnectionError):
        logger.warning(f'公众号爬取出现网络异常：`{url}')
    except:
        logger.warning(f'公众号爬取出现未知异常：`{url}') 

# 需要导入模块: import requests [as 别名]
# 或者: from requests import ReadTimeout [as 别名]
def save_avatar(avatar, userid, size=100):
    """
    保存网络头像
    :param avatar:
    :param userid:
    :param size:
    :return: 保存后的头像地址
    """
    try:
        rsp = requests.get(avatar, timeout=10)

        if rsp.ok:
            img_obj = Image.open(BytesIO(rsp.content))
            img_obj.thumbnail((size, size))
            jpg = get_hash_name(userid) + '.jpg'

            if img_obj.mode != 'RGB':
                img_obj = img_obj.convert('RGB')

            img_obj.save(os.path.join(settings.BASE_DIR, 'assets', 'avatar', jpg))
            return f'/assets/avatar/{jpg}'
        else:
            logger.error(f"同步用户头像出现网络异常！`{userid}`{avatar}")
    except (ConnectTimeout, HTTPError, ReadTimeout, Timeout, ConnectionError):
        logger.error(f"同步用户头像网络异常！`{userid}`{avatar}")
    except:
        logger.error(f"同步用户头像未知异常`{userid}`{avatar}")

    return '/assets/img/logo.svg' 

# 需要导入模块: import requests [as 别名]
# 或者: from requests import ReadTimeout [as 别名]
def request(self, method, url, **kwargs):
        retry = kwargs.pop("retry", True)
        try:
            url = self.__client.path2url(
                url)  # may raise adbutils.AdbError when device offline
            return super().request(method, url, **kwargs)
        except (requests.ConnectionError, requests.ReadTimeout,
                adbutils.AdbError) as e:
            if not retry:
                raise

            # if atx-agent is already running, just raise error
            if isinstance(e, requests.RequestException) and \
                self.__client._is_agent_alive():
                raise

        if not self.__client._serial:
            raise EnvironmentError(
                "http-request to atx-agent error, can only recover from USB")

        logger.warning("atx-agent has something wrong, auto recovering")
        # ReadTimeout: sometime means atx-agent is running but not responsing
        # one reason is futex_wait_queue: https://stackoverflow.com/questions/9801256/app-hangs-on-futex-wait-queue-me-every-a-couple-of-minutes

        # fix atx-agent and request again
        self.__client._prepare_atx_agent()
        url = self.__client.path2url(url)
        return super().request(method, url, **kwargs) 

# 需要导入模块: import requests [as 别名]
# 或者: from requests import ReadTimeout [as 别名]
def _jsonrpc_retry_call(self, *args, **kwargs):
        try:
            return self._jsonrpc_call(*args, **kwargs)
        except (requests.ReadTimeout,
                ServerError,
                UiAutomationNotConnectedError) as e:
            self.reset_uiautomator(str(e)) # uiautomator可能出问题了，强制重启一下
        except (NullObjectExceptionError,
                NullPointerExceptionError,
                StaleObjectExceptionError) as e:
            logger.warning("jsonrpc call got: %s", str(e))
        return self._jsonrpc_call(*args, **kwargs) 

# 需要导入模块: import requests [as 别名]
# 或者: from requests import ReadTimeout [as 别名]
def _is_alive(self):
        try:
            r = self.http.post("/jsonrpc/0", timeout=2, retry=False, data=json.dumps({
                "jsonrpc": "2.0",
                "id": 1,
                "method": "deviceInfo",
            }))
            if r.status_code != 200:
                return False
            if r.json().get("error"):
                return False
            return True
        except (requests.ReadTimeout, EnvironmentError):
            return False 

# 需要导入模块: import requests [as 别名]
# 或者: from requests import ReadTimeout [as 别名]
def wait(self, exists=True, timeout=None):
        """
        Wait until UI Element exists or gone

        Args:
            timeout (float): wait element timeout

        Example:
            d(text="Clock").wait()
            d(text="Settings").wait("gone") # wait until it's gone
        """
        if timeout is None:
            timeout = self.wait_timeout
        http_wait = timeout + 10
        if exists:
            try:
                return self.jsonrpc.waitForExists(self.selector,
                                                  int(timeout * 1000),
                                                  http_timeout=http_wait)
            except requests.ReadTimeout as e:
                warnings.warn("waitForExists readTimeout: %s" % e,
                              RuntimeWarning)
                return self.exists()
        else:
            try:
                return self.jsonrpc.waitUntilGone(self.selector,
                                                  int(timeout * 1000),
                                                  http_timeout=http_wait)
            except requests.ReadTimeout as e:
                warnings.warn("waitForExists readTimeout: %s" % e,
                              RuntimeWarning)
                return not self.exists() 

# 需要导入模块: import requests [as 别名]
# 或者: from requests import ReadTimeout [as 别名]
def _get_member_by_mid(self, mid: int) -> Optional[dict]:
        """
        根据用户id获取其信息
        :param mid: B站用户id
        :return: 用户详情 or None
        """
        get_params = {
            'mid': mid,
            'jsonp': 'jsonp'
        }
        try:
            res_json = requests.get(API_MEMBER_INFO, params=get_params, timeout=WAIT_MAX, proxies=self.cur_proxy,
                                    headers=self.headers).json()
        except ConnectTimeout as e:
            print(f'获取用户id: {mid} 详情失败: 请求接口超时, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except ReadTimeout as e:
            print(f'获取用户id: {mid} 详情失败: 接口读取超时, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except ValueError as e:
            # 解析json失败基本上就是ip被封了
            print(f'获取用户id: {mid} 详情失败: 解析json出错, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except ProxyError as e:
            print(f'获取用户id: {mid} 详情失败: 连接代理失败, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except requests.ConnectionError as e:
            # 可以断定就是代理IP地址无效
            print(f'获取用户id: {mid} 详情失败: 连接错误, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        except ChunkedEncodingError as e:
            print(f'获取用户id: {mid} 详情失败: 远程主机强迫关闭了一个现有的连接, 当前代理:{self.cur_proxy["https"]}')
            raise RequestException(str(e))
        else:
            if res_json['code'] == -404:
                print(f'找不到用户mid:{mid}')
                raise UserNotFoundException(f'找不到用户mid:{mid}')
            if 'data' in res_json:
                return res_json['data']
            print(f'获取用户id: {mid} 详情失败: data字段不存在!')
        return 

# 需要导入模块: import requests [as 别名]
# 或者: from requests import ReadTimeout [as 别名]
def need_retry(exception):
    """
    need to retry
    :param exception:
    :return:
    """
    result = isinstance(exception, (requests.ConnectionError, requests.ReadTimeout))
    if result:
        print('Exception', type(exception), 'occurred, retrying...')
    return result