Python tenacity.retry方法代码示例

# 需要导入模块: import tenacity [as 别名]
# 或者: from tenacity import retry [as 别名]
def _request(session, method, url, params=None, data=None):
    """Request with params.

    Easy to use requests and auto retry.

    Args:
        session: requests session, login session.
        method: string, 'POST' OR 'GET'.
        url: string, post url.
        params=None: dict, get param.
        data=None: dict, post param.

    Returns:
        requests request.

    Raises:
        AutomataError: method param error.
    """
    if method not in ['POST', 'GET'] or not session:
        raise AutomataError

    req = session.request(method, url, params=params, data=data)
    return req.text 

# 需要导入模块: import tenacity [as 别名]
# 或者: from tenacity import retry [as 别名]
def quota_retry(*args, **kwargs) -> Callable:
        """
        A decorator that provides a mechanism to repeat requests in response to exceeding a temporary quote
        limit.
        """
        def decorator(fun: Callable):
            default_kwargs = {
                'wait': tenacity.wait_exponential(multiplier=1, max=100),
                'retry': retry_if_temporary_quota(),
                'before': tenacity.before_log(log, logging.DEBUG),
                'after': tenacity.after_log(log, logging.DEBUG),
            }
            default_kwargs.update(**kwargs)
            return tenacity.retry(
                *args, **default_kwargs
            )(fun)
        return decorator 

# 需要导入模块: import tenacity [as 别名]
# 或者: from tenacity import retry [as 别名]
def operation_in_progress_retry(*args, **kwargs) -> Callable:
        """
        A decorator that provides a mechanism to repeat requests in response to
        operation in progress (HTTP 409)
        limit.
        """
        def decorator(fun: Callable):
            default_kwargs = {
                'wait': tenacity.wait_exponential(multiplier=1, max=300),
                'retry': retry_if_operation_in_progress(),
                'before': tenacity.before_log(log, logging.DEBUG),
                'after': tenacity.after_log(log, logging.DEBUG),
            }
            default_kwargs.update(**kwargs)
            return tenacity.retry(
                *args, **default_kwargs
            )(fun)
        return decorator 

# 需要导入模块: import tenacity [as 别名]
# 或者: from tenacity import retry [as 别名]
def _fetch(self, url, verb='get', **kwargs):
        """return response or None in case of failure, try twice"""
        @retry(stop=stop_after_attempt(2), wait=wait_fixed(2))
        def _inner_fetch(verb='get'):
            headers = {
                'Authorization': 'apiToken %s' % C.DEFAULT_SHIPPABLE_TOKEN
            }

            logging.info(u'%s %s' % (verb, url))
            http_method = getattr(requests, verb)
            resp = http_method(url, headers=headers, **kwargs)
            logging.info(u'shippable status code: %s' % resp.status_code)
            logging.info(u'shippable reason: %s' % resp.reason)

            if resp.status_code not in [200, 302, 400]:
                logging.error(u'RC: %s', resp.status_code)
                raise TryAgain

            return resp

        try:
            logging.debug(u'%s' % url)
            return _inner_fetch(verb=verb)
        except RetryError as e:
            logging.error(e) 

# 需要导入模块: import tenacity [as 别名]
# 或者: from tenacity import retry [as 别名]
def download_from_s3(file_info, context, is_raw_frame=False):
    frame_id = file_info.get('frameid')

    logger.info(f"Downloading file {file_info.get('filename')} from archive. ID: {frame_id}.",
                extra_tags={'filename': file_info.get('filename'),
                            'attempt_number': download_from_s3.retry.statistics['attempt_number']})

    if is_raw_frame:
        url = f'{context.RAW_DATA_FRAME_URL}/{frame_id}'
        archive_auth_token = context.RAW_DATA_AUTH_TOKEN
    else:
        url = f'{context.ARCHIVE_FRAME_URL}/{frame_id}'
        archive_auth_token = context.ARCHIVE_AUTH_TOKEN
    response = requests.get(url, headers=archive_auth_token).json()
    buffer = io.BytesIO()
    buffer.write(requests.get(response['url'], stream=True).content)
    buffer.seek(0)
    return buffer 

# 需要导入模块: import tenacity [as 别名]
# 或者: from tenacity import retry [as 别名]
def test_gc_stresstest():
  with Storage('gs://seunglab-test/cloudvolume/connection_pool/', n_threads=0) as stor:
    stor.put_file('test', 'some string')

  n_trials = 500
  pbar = tqdm(total=n_trials)

  @retry
  def create_conn(interface):
    # assert GC_POOL.total_connections() <= GC_POOL.max_connections * 5
    bucket = GC_POOL.get_connection()
    blob = bucket.get_blob('cloudvolume/connection_pool/test')
    blob.download_as_string()
    GC_POOL.release_connection(bucket)
    pbar.update()

  with ThreadedQueue(n_threads=20) as tq:
    for _ in range(n_trials):
      tq.put(create_conn)

  pbar.close() 

# 需要导入模块: import tenacity [as 别名]
# 或者: from tenacity import retry [as 别名]
def test_s3_stresstest():
  with Storage('s3://seunglab-test/cloudvolume/connection_pool/', n_threads=0) as stor:
    stor.put_file('test', 'some string')

  n_trials = 500
  pbar = tqdm(total=n_trials)

  @retry
  def create_conn(interface):
    conn = S3_POOL.get_connection()  
    # assert S3_POOL.total_connections() <= S3_POOL.max_connections * 5
    bucket = conn.get_object(
      Bucket='seunglab-test',
      Key='cloudvolume/connection_pool/test',
    )
    S3_POOL.release_connection(conn)
    pbar.update()

  with ThreadedQueue(n_threads=20) as tq:
    for _ in range(n_trials):
      tq.put(create_conn)

  pbar.close() 

# 需要导入模块: import tenacity [as 别名]
# 或者: from tenacity import retry [as 别名]
def clone_git_repo(self, repo_url, repo_dir, rev="origin/branches/default/tip"):
        logger.info(f"Cloning {repo_url}...")

        if not os.path.exists(repo_dir):
            tenacity.retry(
                wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
                stop=tenacity.stop_after_attempt(5),
            )(
                lambda: subprocess.run(
                    ["git", "clone", "--quiet", repo_url, repo_dir], check=True
                )
            )()

        tenacity.retry(
            wait=tenacity.wait_exponential(multiplier=1, min=16, max=64),
            stop=tenacity.stop_after_attempt(5),
        )(
            lambda: subprocess.run(
                ["git", "fetch"], cwd=repo_dir, capture_output=True, check=True,
            )
        )()

        subprocess.run(
            ["git", "checkout", rev], cwd=repo_dir, capture_output=True, check=True
        ) 

# 需要导入模块: import tenacity [as 别名]
# 或者: from tenacity import retry [as 别名]
def test_attempt_number_is_correct_for_interleaved_coroutines(self):

        attempts = []

        def after(retry_state):
            attempts.append((retry_state.args[0], retry_state.attempt_number))

        thing1 = NoIOErrorAfterCount(3)
        thing2 = NoIOErrorAfterCount(3)

        await asyncio.gather(
            _retryable_coroutine.retry_with(after=after)(thing1),
            _retryable_coroutine.retry_with(after=after)(thing2))

        # There's no waiting on retry, only a wait in the coroutine, so the
        # executions should be interleaved.
        even_thing_attempts = attempts[::2]
        things, attempt_nos1 = zip(*even_thing_attempts)
        assert len(set(things)) == 1
        assert list(attempt_nos1) == [1, 2, 3]

        odd_thing_attempts = attempts[1::2]
        things, attempt_nos2 = zip(*odd_thing_attempts)
        assert len(set(things)) == 1
        assert list(attempt_nos2) == [1, 2, 3] 

# 需要导入模块: import tenacity [as 别名]
# 或者: from tenacity import retry [as 别名]
def _do_it_with_persistence(func, args, config):
    """Exec func with retries based on provided cli flags

    :param: func: function to attempt to execute
    :param: args: argparser generated cli arguments
    :param: config: configparser object of vaultlocker config
    """
    @tenacity.retry(
        wait=tenacity.wait_fixed(1),
        reraise=True,
        stop=(
            tenacity.stop_after_delay(args.retry) if args.retry > 0
            else tenacity.stop_after_attempt(1)
            ),
        retry=(
            tenacity.retry_if_exception(hvac.exceptions.VaultNotInitialized) |
            tenacity.retry_if_exception(hvac.exceptions.VaultDown)
            )
        )
    def _do_it():
        client = _vault_client(config)
        func(args, client, config)
    _do_it() 

# 需要导入模块: import tenacity [as 别名]
# 或者: from tenacity import retry [as 别名]
def get_connection_from_config(conf):
    retries = conf.database.max_retries
    url = conf.database.connection
    connection_scheme = urlparse.urlparse(url).scheme
    LOG.debug('looking for %(name)r driver in %(namespace)r',
              {'name': connection_scheme, 'namespace': _NAMESPACE})
    mgr = driver.DriverManager(_NAMESPACE, connection_scheme)

    @tenacity.retry(
        wait=tenacity.wait_fixed(conf.database.retry_interval),
        stop=tenacity.stop_after_attempt(retries if retries >= 0 else 5),
        reraise=True)
    def _get_connection():
        """Return an open connection to the database."""
        return mgr.driver(conf, url)

    return _get_connection() 

# 需要导入模块: import tenacity [as 别名]
# 或者: from tenacity import retry [as 别名]
def _action(action, request_session, *args, **kwargs):
        """ Perform a session action and retry if auth fails

        This function dynamically performs a specific type of call
        using the provided session (get, patch, post, etc). It will
        attempt a single re-authentication if the initial request
        fails with a 401.
        """
        _action = getattr(request_session, action)
        try:
            req = _action(*args, **kwargs)
            RegistrySessionHelper.check_status(session=request_session,
                                               request=req)
        except requests.exceptions.HTTPError as e:
            if e.response.status_code == 401:
                req = _action(*args, **kwargs)
                RegistrySessionHelper.check_status(session=request_session,
                                                   request=req)
            else:
                raise
        return req 

# 需要导入模块: import tenacity [as 别名]
# 或者: from tenacity import retry [as 别名]
def request(self, method, url, *a, **kw):
        def _after_request_attempt(func, tries, secs):
            msg = f'RETRY {tries} attempt(s)'
            if tries > 1:
                interval = humanize_interval(secs)
                msg += f'. Time spent: {interval}'
            msg += ' ...'
            self.logger.warning(msg)

        if self.settings.get('RETRY_ENABLED'):
            retry_kw = {'after': _after_request_attempt}
            retry_kw.update(self.settings.get('RETRY_SETTINGS', {}))
        else:
            retry_kw = {'retry': retry_never}

        @retry(**retry_kw)
        def _request():
            with self.rate_limiter:
                self.logger.info(f'{method} "{url}"')
                response = self._requests.request(method, url, *a, **kw)
                self._update_csrftoken(response)
                return validate_response(self, response)

        return _request() 

# 需要导入模块: import tenacity [as 别名]
# 或者: from tenacity import retry [as 别名]
def etcd(self):
        endpoint = os.environ.get('PYTHON_ETCD_HTTP_URL')
        timeout = 5
        if endpoint:
            url = urlparse(endpoint)
            with etcd3.client(host=url.hostname,
                              port=url.port,
                              timeout=timeout) as client:
                yield client
        else:
            with etcd3.client() as client:
                yield client

        @retry(wait=wait_fixed(2), stop=stop_after_attempt(3))
        def delete_keys_definitely():
            # clean up after fixture goes out of scope
            etcdctl('del', '--prefix', '/')
            out = etcdctl('get', '--prefix', '/')
            assert 'kvs' not in out

        delete_keys_definitely() 

# 需要导入模块: import tenacity [as 别名]
# 或者: from tenacity import retry [as 别名]
def get(details=True):
        enforce("get status", {})
        try:
            members_req = pecan.request.coordinator.get_members(
                metricd.MetricProcessor.GROUP_ID)
        except tooz.NotImplemented:
            members_req = None
        try:
            report = pecan.request.incoming.measures_report(
                strtobool("details", details))
        except incoming.ReportGenerationError:
            abort(503, 'Unable to generate status. Please retry.')
        report_dict = {"storage": {"summary": report['summary']}}
        if 'details' in report:
            report_dict["storage"]["measures_to_process"] = report['details']
        report_dict['metricd'] = {}
        if members_req:
            members = members_req.get()
            caps = [
                pecan.request.coordinator.get_member_capabilities(
                    metricd.MetricProcessor.GROUP_ID, member)
                for member in members
            ]
            report_dict['metricd']['processors'] = [
                member.decode() for member in members
            ]
            report_dict['metricd']['statistics'] = {
                member.decode(): cap.get()
                for member, cap in six.moves.zip(members, caps)
            }
        else:
            report_dict['metricd']['processors'] = None
            report_dict['metricd']['statistics'] = {}
        return report_dict