Python apiproxy_errors.DeadlineExceededError方法代码示例

# 需要导入模块: from google.appengine.runtime import apiproxy_errors [as 别名]
# 或者: from google.appengine.runtime.apiproxy_errors import DeadlineExceededError [as 别名]
def get(self):
        """Indefinitely fetch tasks and update the datastore."""
        queue = taskqueue.Queue('pullq')
        while True:
            try:
                tasks = queue.lease_tasks_by_tag(3600, 1000, deadline=60)
            except (taskqueue.TransientError,
                    apiproxy_errors.DeadlineExceededError) as e:
                logging.exception(e)
                time.sleep(1)
                continue

            if tasks:
                key = tasks[0].tag

                try:
                    update_counter(key, tasks)
                except Exception as e:
                    logging.exception(e)
                    raise
                finally:
                    queue.delete_tasks(tasks)

            time.sleep(1) 

# 需要导入模块: from google.appengine.runtime import apiproxy_errors [as 别名]
# 或者: from google.appengine.runtime.apiproxy_errors import DeadlineExceededError [as 别名]
def get_own_public_certificates():
  """Returns CertificateBundle with certificates of the current service."""
  attempt = 0
  while True:
    attempt += 1
    try:
      certs = app_identity.get_public_certificates(deadline=1.5)
      break
    except apiproxy_errors.DeadlineExceededError as e:
      logging.warning('%s', e)
      if attempt == 3:
        raise
  return CertificateBundle({
    'app_id': app_identity.get_application_id(),
    'service_account_name': utils.get_service_account_name(),
    'certificates': [
      {
        'key_name': cert.key_name,
        'x509_certificate_pem': cert.x509_certificate_pem,
      }
      for cert in certs
    ],
    'timestamp': utils.datetime_to_timestamp(utils.utcnow()),
  }) 

# 需要导入模块: from google.appengine.runtime import apiproxy_errors [as 别名]
# 或者: from google.appengine.runtime.apiproxy_errors import DeadlineExceededError [as 别名]
def _WaitImpl(self):
    """Waits on the API call associated with this RPC. The callback,
    if provided, will be executed before Wait() returns. If this RPC
    is already complete, or if the RPC was never started, this
    function will return immediately.

    Raises:
      InterruptedError if a callback throws an uncaught exception.
    """
    try:
      rpc_completed = _apphosting_runtime___python__apiproxy.Wait(self)
    except (runtime.DeadlineExceededError, apiproxy_errors.InterruptedError):
      raise
    except:
      exc_class, exc, tb = sys.exc_info()
      if (isinstance(exc, SystemError) and
          exc.args[0] == 'uncaught RPC exception'):
        raise
      rpc = None
      if hasattr(exc, "_appengine_apiproxy_rpc"):
        rpc = exc._appengine_apiproxy_rpc

      new_exc = apiproxy_errors.InterruptedError(exc, rpc)
      raise new_exc.__class__, new_exc, tb
    return True 

# 需要导入模块: from google.appengine.runtime import apiproxy_errors [as 别名]
# 或者: from google.appengine.runtime.apiproxy_errors import DeadlineExceededError [as 别名]
def GetBackups(self, limit=100, deadline=10):
    """Obtain a list of backups.

    Args:
      limit: maximum number of backup records to retrieve.
      deadline: maximum number of seconds to spend getting backups.

    Returns:
      List of backups, sorted in reverse order by completion time.
    """
    backups = []
    query = backup_handler.BackupInformation.all()
    query.filter('complete_time > ', 0)
    query.order('-complete_time')
    try:
      backups.extend(query.run(deadline=deadline, limit=limit))
    except (datastore_errors.Timeout, apiproxy_errors.DeadlineExceededError):
      logging.warning('Failed to retrieve all backups within deadline.')
    return backups 

# 需要导入模块: from google.appengine.runtime import apiproxy_errors [as 别名]
# 或者: from google.appengine.runtime.apiproxy_errors import DeadlineExceededError [as 别名]
def _get_memory_usage():
  """Returns the amount of memory used as an float in MiB."""
  try:
    return apiruntime.runtime.memory_usage().current()
  except (AssertionError,
          apiproxy_errors.CancelledError,
          apiproxy_errors.DeadlineExceededError,
          apiproxy_errors.RPCFailedError,
          runtime.DeadlineExceededError) as e:
    logging.warning('Failed to get memory usage: %s', e)
    return None


## Handler 

# 需要导入模块: from google.appengine.runtime import apiproxy_errors [as 别名]
# 或者: from google.appengine.runtime.apiproxy_errors import DeadlineExceededError [as 别名]
def report_memory(app):
  """Wraps an app so handlers log when memory usage increased by at least 0.5MB
  after the handler completed.
  """
  min_delta = 0.5
  old_dispatcher = app.router.dispatch
  def dispatch_and_report(*args, **kwargs):
    before = _get_memory_usage()
    deadline = False
    try:
      return old_dispatcher(*args, **kwargs)
    except runtime.DeadlineExceededError:
      # Don't try to call any function after, it'll likely fail anyway. It is
      # because _get_memory_usage() does an RPC under the hood.
      deadline = True
      raise
    finally:
      if not deadline:
        after = _get_memory_usage()
        if before and after and after >= before + min_delta:
          logging.debug(
              'Memory usage: %.1f -> %.1f MB; delta: %.1f MB',
              before, after, after-before)
  app.router.dispatch = dispatch_and_report


## Time 

# 需要导入模块: from google.appengine.runtime import apiproxy_errors [as 别名]
# 或者: from google.appengine.runtime.apiproxy_errors import DeadlineExceededError [as 别名]
def _fetch_service_certs(service_url):
  protocol = 'https://'
  if utils.is_local_dev_server():
    protocol = ('http://', 'https://')
  assert service_url.startswith(protocol), (service_url, protocol)
  url = '%s/auth/api/v1/server/certificates' % service_url

  # Retry code is adapted from components/net.py. net.py can't be used directly
  # since it depends on components.auth (and dependency cycles between
  # components are bad).
  attempt = 0
  result = None
  while attempt < 4:
    if attempt:
      logging.info('Retrying...')
    attempt += 1
    logging.info('GET %s', url)
    try:
      result = urlfetch.fetch(
          url=url,
          method='GET',
          headers={'X-URLFetch-Service-Id': utils.get_urlfetch_service_id()},
          follow_redirects=False,
          deadline=5,
          validate_certificate=True)
    except (apiproxy_errors.DeadlineExceededError, urlfetch.Error) as e:
      # Transient network error or URL fetch service RPC deadline.
      logging.warning('GET %s failed: %s', url, e)
      continue
    # It MUST return 200 on success, it can't return 403, 404 or >=500.
    if result.status_code != 200:
      logging.warning(
          'GET %s failed, HTTP %d: %r', url, result.status_code, result.content)
      continue
    return json.loads(result.content)

  # All attempts failed, give up.
  msg = 'Failed to grab public certs from %s (HTTP code %s)' % (
      service_url, result.status_code if result else '???')
  raise CertificateError(msg, transient=True) 

# 需要导入模块: from google.appengine.runtime import apiproxy_errors [as 别名]
# 或者: from google.appengine.runtime.apiproxy_errors import DeadlineExceededError [as 别名]
def attempt_oauth_initialization(scope):
  """Attempts to perform GetOAuthUser RPC retrying deadlines.

  The result it cached in appengine.api.oauth guts. Never raises exceptions,
  just gives up letting subsequent oauth.* calls fail in a proper way.
  """
  # 4 attempts: ~20 sec (default RPC deadline is 5 sec).
  attempt = 0
  while attempt < 4:
    attempt += 1
    try:
      oauth.get_client_id(scope)
      return
    except apiproxy_errors.DeadlineExceededError as e:
      logging.warning('DeadlineExceededError: %s', e)
      continue
    except oauth.OAuthServiceFailureError as e:
      logging.warning(
          'oauth.OAuthServiceFailureError (%s): %s', e.__class__.__name__, e)
      # oauth library "caches" the error code in os.environ and retrying
      # oauth.get_client_id doesn't do anything. Clear this cache first, see
      # oauth_api.py, _maybe_call_get_oauth_user in GAE SDK.
      os.environ.pop('OAUTH_ERROR_CODE', None)
      continue
    except oauth.Error as e:
      # Next call to oauth.get_client_id() will trigger same error and it will
      # be handled for real.
      logging.warning('oauth.Error (%s): %s', e.__class__.__name__, e)
      return 

# 需要导入模块: from google.appengine.runtime import apiproxy_errors [as 别名]
# 或者: from google.appengine.runtime.apiproxy_errors import DeadlineExceededError [as 别名]
def GetKindsForCurrentNamespace(deadline):
  """Obtain a list of all kind names from the datastore.

  Pulls kinds from the current namespace only. The result is alphabetized.

  Args:
    deadline: maximum number of seconds to spend getting kinds.

  Returns:
    kinds: an alphabetized list of kinds for the specified namespace(s).
    more_kinds: a boolean indicating whether there may be additional kinds
        not included in 'kinds' (e.g. because the query limit was reached).
  """
  more_kinds = False
  kind_names = []
  try:
    kinds = metadata.Kind.all().order('__key__').run(batch_size=1000,
                                                     deadline=deadline)
    for kind in kinds:
      kind_name = kind.kind_name
      if IsKindNameVisible(kind_name):
        kind_names.append(kind_name)
  except (datastore_errors.Timeout, apiproxy_errors.DeadlineExceededError):
    more_kinds = True
    logging.warning('Failed to retrieve all kinds within deadline.')
  return kind_names, more_kinds 

# 需要导入模块: from google.appengine.runtime import apiproxy_errors [as 别名]
# 或者: from google.appengine.runtime.apiproxy_errors import DeadlineExceededError [as 别名]
def PerformWork(self, thread_pool):
    """Perform the work of this work item and report the results.

    Args:
      thread_pool: An AdaptiveThreadPool instance.

    Returns:
      A tuple (status, instruction) of the work status and an instruction
      for the ThreadGate.
    """
    status = adaptive_thread_pool.WorkItem.FAILURE
    instruction = adaptive_thread_pool.ThreadGate.DECREASE


    try:
      self.MarkAsTransferring()



      try:
        transfer_time = self._TransferItem(thread_pool)
        if transfer_time is None:
          status = adaptive_thread_pool.WorkItem.RETRY
          instruction = adaptive_thread_pool.ThreadGate.HOLD
        else:
          logger.debug('[%s] %s Transferred %d entities in %0.1f seconds',
                       threading.currentThread().getName(), self, self.count,
                       transfer_time)
          sys.stdout.write('.')
          sys.stdout.flush()
          status = adaptive_thread_pool.WorkItem.SUCCESS
          if transfer_time <= MAXIMUM_INCREASE_DURATION:
            instruction = adaptive_thread_pool.ThreadGate.INCREASE
          elif transfer_time <= MAXIMUM_HOLD_DURATION:
            instruction = adaptive_thread_pool.ThreadGate.HOLD
      except (db.InternalError, db.NotSavedError, db.Timeout,
              db.TransactionFailedError,
              apiproxy_errors.OverQuotaError,
              apiproxy_errors.DeadlineExceededError,
              apiproxy_errors.ApplicationError), e:

        status = adaptive_thread_pool.WorkItem.RETRY
        logger.exception('Retrying on non-fatal datastore error: %s', e)
      except urllib2.HTTPError, e:
        http_status = e.code
        if http_status >= 500 and http_status < 600:

          status = adaptive_thread_pool.WorkItem.RETRY
          logger.exception('Retrying on non-fatal HTTP error: %d %s',
                           http_status, e.msg)
        else:
          self.SetError()
          status = adaptive_thread_pool.WorkItem.FAILURE 

# 需要导入模块: from google.appengine.runtime import apiproxy_errors [as 别名]
# 或者: from google.appengine.runtime.apiproxy_errors import DeadlineExceededError [as 别名]
def _fetch_certs_from_json(url, service_account_name=None):
  """Fetches certs from a JSON bundle in form {<key_id>: <pem encoded cert>}."""
  # Retry code is adapted from components/net.py. net.py can't be used directly
  # since it depends on components.auth (and dependency cycles between
  # components are bad).
  attempt = 0
  result = None
  while attempt < 4:
    if attempt:
      logging.info('Retrying...')
    attempt += 1
    logging.info('GET %s', url)
    try:
      result = urlfetch.fetch(
          url=url,
          method='GET',
          follow_redirects=False,
          deadline=5,
          validate_certificate=True)
    except (apiproxy_errors.DeadlineExceededError, urlfetch.Error) as e:
      # Transient network error or URL fetch service RPC deadline.
      logging.warning('GET %s failed: %s', url, e)
      continue
    # It MUST return 200 on success, it can't return 403, 404 or >=500.
    if result.status_code != 200:
      logging.warning(
          'GET %s failed, HTTP %d: %r', url, result.status_code, result.content)
      continue
    response = json.loads(result.content)
    return {
      'service_account_name': service_account_name,
      'certificates': [
        {
          'key_name': key_name,
          'x509_certificate_pem': pem,
        }
        for key_name, pem in sorted(response.items())
      ],
      'timestamp': utils.datetime_to_timestamp(utils.utcnow()),
    }

  # All attempts failed, give up.
  msg = 'Failed to grab service account certs for %s (HTTP code %s)' % (
      service_account_name, result.status_code if result else '???')
  raise CertificateError(msg, transient=True) 

# 需要导入模块: from google.appengine.runtime import apiproxy_errors [as 别名]
# 或者: from google.appengine.runtime.apiproxy_errors import DeadlineExceededError [as 别名]
def _call_async(url, payload, method, headers):
  """Makes URL fetch call aggressively retrying on errors a bunch of times.

  On success returns deserialized JSON response body.
  On failure raises AccessTokenError.
  """
  attempt = 0
  while attempt < 4:
    if attempt:
      logging.info('Retrying...')
    attempt += 1
    logging.info('%s %s', method, url)
    try:
      response = yield _urlfetch(
          url=url,
          payload=payload,
          method=method,
          headers=headers,
          follow_redirects=False,
          deadline=5,  # all RPCs we do should be fast
          validate_certificate=True)
    except (apiproxy_errors.DeadlineExceededError, urlfetch.Error) as e:
      # Transient network error or URL fetch service RPC deadline.
      logging.warning('%s %s failed: %s', method, url, e)
      continue

    # Transient error on the other side.
    if response.status_code >= 500:
      logging.warning(
          '%s %s failed with HTTP %d: %r',
          method, url, response.status_code, response.content)
      continue

    # Non-transient error.
    if 300 <= response.status_code < 500:
      logging.warning(
          '%s %s failed with HTTP %d: %r',
          method, url, response.status_code, response.content)
      raise AccessTokenError(
          'Failed to call %s: HTTP %d' % (url, response.status_code))

    # Success.
    try:
      body = json.loads(response.content)
    except ValueError:
      logging.error('Non-JSON response from %s: %r', url, response.content)
      raise AccessTokenError('Non-JSON response from %s' % url)
    raise ndb.Return(body)

  # All our attempts failed with transient errors. Perhaps some later retry
  # can help, so set transient to True.
  raise AccessTokenError(
      'Failed to call %s after multiple attempts' % url, transient=True)