Python retry.Retry方法代码示例

# 需要导入模块: from urllib3.util import retry [as 别名]
# 或者: from urllib3.util.retry import Retry [as 别名]
def find_bad_items(self, batch: Mapping[str, DatabaseMedia]):
        """
        a batch get failed. Now do all of its contents as individual
        gets so we can work out which ID(s) cause the failure
        """
        for item_id, media_item in batch.items():
            try:
                log.debug("BAD ID Retry on %s (%s)", item_id, media_item.relative_path)
                response = self._api.mediaItems.get.execute(mediaItemId=item_id)
                media_item_json = response.json()
                self.download_file(media_item, media_item_json)
            except RequestException as e:
                self.bad_ids.add_id(
                    str(media_item.relative_path), media_item.id, media_item.url, e
                )
                self.files_download_failed += 1
                log.error(
                    "FAILURE %d in get of %s BAD ID",
                    self.files_download_failed,
                    media_item.relative_path,
                ) 

# 需要导入模块: from urllib3.util import retry [as 别名]
# 或者: from urllib3.util.retry import Retry [as 别名]
def retry_session(retries=3, backoff_factor=0.8, status_forcelist=(500, 502, 504)):
    """Add retry to Requests Session.

    https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.retry.Retry
    """
    session = Session()
    retries = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    # mount all https requests
    session.mount('https://', HTTPAdapter(max_retries=retries))
    return session 

# 需要导入模块: from urllib3.util import retry [as 别名]
# 或者: from urllib3.util.retry import Retry [as 别名]
def check_github_users(usernames, retries=5, backoff_factor=0.3, status_forcelist=(500, 502, 503, 504)):
    """
    Check if provided usernames exist in Github
    :param usernames: list of usernames
    :param retries: number of retries
    :param backoff_factor: backoff to apply between retries
    :param status_forcelist: HTTP status codes that should be retried
    :return: list of usernames that exist in Github
    """
    session = requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist,
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount('https://', adapter)
    return [
        username for username in usernames
        if session.get('https://github.com/{}.keys'.format(username)).status_code == 200
    ] 

# 需要导入模块: from urllib3.util import retry [as 别名]
# 或者: from urllib3.util.retry import Retry [as 别名]
def retry(
    total_requests=10, backoff_factor=1, statuses=(500, 502, 503, 504, 429)
):
    """Default HTTP session for requests."""
    _session = requests.Session()

    retries = Retry(
        total=total_requests,
        backoff_factor=backoff_factor,
        status_forcelist=list(statuses)
    )

    _session.mount('http://', HTTPAdapter(max_retries=retries))
    _session.mount('https://', HTTPAdapter(max_retries=retries))

    yield _session 

# 需要导入模块: from urllib3.util import retry [as 别名]
# 或者: from urllib3.util.retry import Retry [as 别名]
def requests(self, session: requests.Session):
        """
        A backoff factor to apply between attempts after the second try (most errors are resolved immediately by a
        second try without a delay). urllib3 will sleep for: {backoff factor} * (2 ^ ({number of total retries} - 1))
        seconds. If the backoff_factor is 0.1, then sleep() will sleep for [0.0s, 0.2s, 0.4s, ...] between retries.
        It will never be longer than Retry.BACKOFF_MAX. By default, backoff is disabled (set to 0).
        :param session:
        :return:
        """
        session_r = session or requests.Session()
        max_retries = Retry(
            total=self.retries,
            read=self.retries,
            connect=self.retries,
            backoff_factor=0.3,
            status_forcelist=self.status_retries,
        )
        adapter = HTTPAdapter(max_retries=max_retries)
        session_r.mount('http://', adapter)
        session_r.mount('https://', adapter)
        return session_r 

# 需要导入模块: from urllib3.util import retry [as 别名]
# 或者: from urllib3.util.retry import Retry [as 别名]
def __init__(self, rest_endpoint, user_agent, opts={}):
        # Must have options
        self.user_agent = user_agent
        self.rest_endpoint = rest_endpoint
        if not rest_endpoint.startswith("https"):
            log('Warning: API endpoint must start with "https".', is_error=True)
        if not user_agent:
            log("Warning: User-Agent header must be set.", is_error=True)
        # Options with defaults, overridden by kwargs
        self.verify_ssl = opts.get("verify_ssl", True)
        self.connect_timeout = opts.get("connect_timeout", 5)
        self.read_timeout = opts.get("read_timeout", 20)
        self.max_retries = opts.get("max_retries", 5)
        # Make all API calls share the same session
        self.api_client = requests.Session()
        # Apply a backoff for failing requests, up to self.max_retries
        # 1st waiting will be 0.1s, then 0.2s, 0.4s, etc, following this formula:
        # {backoff factor} * (2 ** ({number of retries so far} - 1))
        retries = Retry(
            total=self.max_retries,
            backoff_factor=0.1,
            status_forcelist=[500, 502, 503, 504],
        )
        a = requests.adapters.HTTPAdapter(max_retries=retries)
        self.api_client.mount("https://", a) 

# 需要导入模块: from urllib3.util import retry [as 别名]
# 或者: from urllib3.util.retry import Retry [as 别名]
def _load_adapter(
        self, max_retries: typing.Optional[int] = None
    ) -> HTTPAdapter:
        if max_retries is None and self._adapter is not None:
            return self._adapter

        max_retries = max_retries or 0
        adapter = HTTPAdapter()
        adapter.max_retries = Retry(
            total=max_retries,
            read=max_retries,
            connect=max_retries,
            backoff_factor=self.backoff_factor,
            status_forcelist=self.status_forcelist,
        )
        return adapter 

# 需要导入模块: from urllib3.util import retry [as 别名]
# 或者: from urllib3.util.retry import Retry [as 别名]
def __init__(self):
        self._config_timestamp = None
        self._mode = None

        self.authorization_endpoint = None
        self.signing_keys = None
        self.token_endpoint = None
        self.end_session_endpoint = None
        self.issuer = None

        method_whitelist = frozenset([
            'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE', 'POST'
        ])

        retry = Retry(
            total=settings.RETRIES,
            read=settings.RETRIES,
            connect=settings.RETRIES,
            backoff_factor=0.3,
            method_whitelist=method_whitelist
        )
        self.session = requests.Session()
        adapter = requests.adapters.HTTPAdapter(max_retries=retry)
        self.session.mount('https://', adapter)
        self.session.verify = settings.CA_BUNDLE 

# 需要导入模块: from urllib3.util import retry [as 别名]
# 或者: from urllib3.util.retry import Retry [as 别名]
def requests_retry_session(retries=5, backoff_factor=0.3, status_forcelist=(500, 502, 504), session=None):
    """

    :param retries:
    :param backoff_factor:
    :param status_forcelist:
    :param session:
    :return:
    """
    s = session or requests.Session()
    retry = Retry(
        total=retries,
        read=retries,
        connect=retries,
        backoff_factor=backoff_factor,
        status_forcelist=status_forcelist)
    adapter = HTTPAdapter(max_retries=retry)
    s.mount('http://', adapter)
    s.mount('https://', adapter)
    return s 

# 需要导入模块: from urllib3.util import retry [as 别名]
# 或者: from urllib3.util.retry import Retry [as 别名]
def __init__(
        self, default_headers=None, max_requests=10, rate_limit=0,
        req_timeout=None, max_retry=10, total_retry=100, drop_404s=False,
    ):
        """Create the MultiRequest.

        Args:
            default_headers - A dict of headers which will be added to every request
            max_requests - Maximum number of requests to issue at once
            rate_limit - Maximum number of requests to issue per second
            req_timeout - Maximum number of seconds to wait without reading a response byte before deciding an error has occurred
            max_retry - The total number of attempts to retry a single batch of requests
            total_retry - The total number of request retries that can be made through the entire session
        Note there is a difference between `max_retry` and `total_retry`:
            - `max_retry` refers to how many times a batch of requests will be re-issued collectively
            - `total_retry` refers to a limit on the total number of outstanding requests made
            Once the latter is exhausted, no failed request within the whole session will be retried.
        """
        self._default_headers = default_headers
        self._max_requests = max_requests
        self._req_timeout = req_timeout or 25.0
        self._max_retry = max_retry
        self._drop_404s = drop_404s
        self._rate_limiter = RateLimiter(rate_limit) if rate_limit else None
        self._availability_limiter = AvailabilityLimiter(total_retry) if total_retry else None
        self._session = FuturesSession(max_workers=max_requests)
        retries = Retry(total=0, status_forcelist=[500, 502, 503, 504], raise_on_status=True)
        self._session.mount(
            'https://', SSLAdapter(
                max_retries=retries, pool_maxsize=max_requests, pool_connections=max_requests,
            ),
        ) 

# 需要导入模块: from urllib3.util import retry [as 别名]
# 或者: from urllib3.util.retry import Retry [as 别名]
def test_retries_500(self):
        retries = 5
        timeout = 2

        session = Session()
        start = datetime.now()
        result = session.get("https://httpbin.org/status/500", timeout=timeout)
        self.assertEqual(result.status_code, 500)
        elapsed = datetime.now() - start

        retry = Retry(
            total=retries,
            backoff_factor=0.1,
            status_forcelist=[500, 502, 503, 504],
            method_whitelist=frozenset(["GET", "POST"]),
            raise_on_status=False,
            respect_retry_after_header=True,
        )

        session.mount("https://", HTTPAdapter(max_retries=retry))

        start = datetime.now()
        result = session.get("https://httpbin.org/status/500", timeout=timeout)
        elapsed2 = datetime.now() - start
        self.assertEqual(result.status_code, 500)
        self.assertGreater(elapsed2, elapsed * (retries - 1)) 

# 需要导入模块: from urllib3.util import retry [as 别名]
# 或者: from urllib3.util.retry import Retry [as 别名]
def test_retries_timeout(self):
        retries = 3
        timeout = 1
        retry_error = False

        session = Session()
        retry = Retry(
            total=retries,
            backoff_factor=0.1,
            status_forcelist=[500, 502, 503, 504],
            method_whitelist=frozenset(["GET", "POST"]),
            raise_on_status=False,
            respect_retry_after_header=True,
        )

        session.mount("https://", HTTPAdapter(max_retries=retry))

        start = datetime.now()
        try:
            _ = session.get("https://httpbin.org/delay/5", timeout=timeout)
        except exceptions.ConnectionError as e:
            retry_error = True
            print(e)

        elapsed = datetime.now() - start
        self.assertEqual(retry_error, True)
        self.assertGreater(elapsed.seconds, retries * timeout) 

# 需要导入模块: from urllib3.util import retry [as 别名]
# 或者: from urllib3.util.retry import Retry [as 别名]
def __init__(self, tcex):
        """Initialize the Class properties."""
        super().__init__()
        self.tcex = tcex

        # properties
        self.args = self.tcex.default_args
        self.auth = None
        self.token = self.tcex.token

        # Update User-Agent
        self.headers.update({'User-Agent': 'TcEx'})

        # Set Proxy
        if self.args.tc_proxy_tc:
            self.proxies = self.tcex.proxies
            self.tcex.log.trace(
                f'Using proxy host {self.args.tc_proxy_host}:'
                f'{self.args.tc_proxy_port} for ThreatConnect API.'
            )

        # Add Retry
        self.retry()

        # Set Verify
        self.verify = self.args.tc_verify 

# 需要导入模块: from urllib3.util import retry [as 别名]
# 或者: from urllib3.util.retry import Retry [as 别名]
def retry(self, retries=3, backoff_factor=0.3, status_forcelist=(500, 502, 504)):
        """Add retry to Requests Session

        https://urllib3.readthedocs.io/en/latest/reference/urllib3.util.html#urllib3.util.retry.Retry
        """
        retries = Retry(
            total=retries,
            read=retries,
            connect=retries,
            backoff_factor=backoff_factor,
            status_forcelist=status_forcelist,
        )
        # mount all https requests
        self.mount('https://', adapters.HTTPAdapter(max_retries=retries)) 

# 需要导入模块: from urllib3.util import retry [as 别名]
# 或者: from urllib3.util.retry import Retry [as 别名]
def renew_token(self, token):
        """Renew expired ThreatConnect Token.

        This method will renew a token and update the token_map with new token and expiration.

        Args:
            token (str): The ThreatConnect API token.
            token_expires (int): The token expiration timestamp.
        """
        api_token_data = {}
        self.log.in_token_renewal = True  # pause API logging

        # log token information
        try:
            params = {'expiredToken': token}
            url = f'{self.token_url}/appAuth'
            r = self.session.get(url, params=params, verify=self.verify)

            if not r.ok:
                err_reason = r.text or r.reason
                err_msg = (
                    f'Token Retry Error. API status code: {r.status_code}, '
                    f'API message: {err_reason}, '
                    f'Token: {self.printable_token(token)}.'
                )
                self.log.error(err_msg)
                raise RuntimeError(1042, err_msg)
        except exceptions.SSLError:  # pragma: no cover
            raise RuntimeError('Token renewal failed with an SSL Error.')

        # process response for token
        try:
            api_token_data = r.json()
        except (AttributeError, ValueError) as e:  # pragma: no cover
            raise RuntimeError(f'Token renewal failed ({e}).')
        finally:
            self.log.in_token_renewal = False

        return api_token_data