Java BackOff.STOP属性代码示例

/**
 * Determines the amount to sleep for and sleeps if needed.
 *
 * @param backoff BackOff to determine how long to sleep for
 * @param sleeper Used to sleep
 * @param currentException exception that caused the retry and sleep. For logging.
 * @throws InterruptedException if sleep is interrupted
 */
private static boolean nextSleep(BackOff backoff, Sleeper sleeper, Exception currentException)
    throws InterruptedException {
  long backOffTime = 0;
  try {
    backOffTime = backoff.nextBackOffMillis();
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
  if (backOffTime == BackOff.STOP) {
    return false;
  }
  LOG.info("Transient exception caught. Sleeping for " + backOffTime + ", then retrying."
      + currentException);
  sleeper.sleep(backOffTime);
  return true;
}
 

void waitBeforeRetry(Exception me) throws Exception {
    if (me instanceof MasterNotFoundException && !noMasterAutoRetry) {
        failureMeter.mark();
        throw me;
    }

    if (me instanceof ConnectionBrokenException && !connectionBrokenAutoRetry) {
        failureMeter.mark();
        throw me;
    }

    try{
        backOffMillis = backoff.nextBackOffMillis();
        if (backOffMillis == BackOff.STOP) {
            failureMeter.mark();
            throw me;
        }
        logger.debug("no master found, auto retry after sleeping {} ms", backOffMillis);
        Thread.sleep(backOffMillis);
    }catch(Exception e) {
    }
}
 

@ProcessElement
public void processElement(DoFn<String, Integer>.ProcessContext context) throws Exception {
  String variantId = context.element();
  // Call the deletion operation via exponential backoff so that "Rate Limit Exceeded"
  // quota issues do not cause the pipeline to fail.
  ExponentialBackOff backoff = new ExponentialBackOff.Builder().build();
  while (true) {
    try {
      genomics.variants().delete(variantId).execute();
      Metrics.counter(DeleteVariantFn.class, "Number of variants deleted").inc();
      context.output(1);
      return;
    } catch (Exception e) {
      if (e.getMessage().startsWith("429 Too Many Requests")) {
        LOG.warn("Backing-off per: ", e);
        long backOffMillis = backoff.nextBackOffMillis();
        if (backOffMillis == BackOff.STOP) {
          throw e;
        }
        Thread.sleep(backOffMillis);
      } else {
        throw e;
      }
    }
  }
}
 

protected <T extends GenericJson> T retryFailures(
    AbstractGoogleClientRequest<T> request, BackOff backOff) throws IOException {
  while (true) {
    try {
      return request.execute();
    } catch (GoogleJsonResponseException e) {
      try {
        long nextPause = backOff.nextBackOffMillis();
        if (nextPause == BackOff.STOP) {
          throw e;
        }
        System.out.printf("Operation failed, retrying in %f seconds.%n", nextPause / 1000.0);
        BackOffUtils.next(Sleeper.DEFAULT, backOff);
      } catch (InterruptedException ie) {
        // Just go straight into retry if interrupted.
      }
    }
  }
}
 

private boolean shouldRetryNow() {
  long backOffMillis;
  try {
    backOffMillis = backoff.nextBackOffMillis();
  } catch (IOException e1) {
    // Something strange happened, just give up.
    backOffMillis = BackOff.STOP;
  }

  if (backOffMillis == BackOff.STOP) {
    backoff.reset();
    return false;
  }

  try {
    Thread.sleep(backOffMillis);
  } catch (InterruptedException e) {
    LOG.log(Level.WARNING, "Backoff sleep interrupted", e);
  }
  return true;
}
 

@Override
public Optional<URI> run(RunSpec runSpec) {
  Sleeper retrySleeper = Sleeper.DEFAULT;
  BackOff backoff = BACKOFF_FACTORY.backoff();

  while (true) {
    try {
      final Pod pod = client.pods().create(createPod(runSpec));
      final String podName = pod.getMetadata().getName();
      LOG.info("Created pod {}", podName);

      Optional<URI> uri = blockUntilComplete(podName);
      client.pods().withName(podName).delete();
      return uri;
    } catch (KubernetesClientException kce) {
      try {
        long sleep = backoff.nextBackOffMillis();
        if (sleep == BackOff.STOP) {
          // Rethrow last error, to be included as a cause in the catch below.
          LOG.error("Failed to create Kubernetes pod", kce);
          throw new KubernetesClientException("Failed to create Kubernetes pod", kce);
        } else {
          LOG.warn("Kubernetes creation attempt failed, sleeping before retrying", kce);
          retrySleeper.sleep(sleep);
        }
      } catch (IOException | InterruptedException ioe) {
        throw new RuntimeException(
            String.format("Failed to create Kubernetes pod when trying to sleep: %s", ioe.getMessage()), ioe);
      }
    } catch (InterruptedException ie) {
      throw new RuntimeException("Interrupted while blocking", ie);
    }
  }
}
 

@Override
public long nextBackOffMillis() {
  // Maximum number of retries reached.
  if (currentRetry >= backoffConfig.maxRetries) {
    return BackOff.STOP;
  }
  // Maximum cumulative backoff reached.
  if (currentCumulativeBackoff.compareTo(backoffConfig.maxCumulativeBackoff) >= 0) {
    return BackOff.STOP;
  }

  double currentIntervalMillis =
      Math.min(
          backoffConfig.initialBackoff.getMillis()
              * Math.pow(backoffConfig.exponent, currentRetry),
          backoffConfig.maxBackoff.getMillis());
  double randomOffset =
      (Math.random() * 2 - 1) * DEFAULT_RANDOMIZATION_FACTOR * currentIntervalMillis;
  long nextBackoffMillis = Math.round(currentIntervalMillis + randomOffset);
  // Cap to limit on cumulative backoff
  Duration remainingCumulative =
      backoffConfig.maxCumulativeBackoff.minus(currentCumulativeBackoff);
  nextBackoffMillis = Math.min(nextBackoffMillis, remainingCumulative.getMillis());

  // Update state and return backoff.
  currentCumulativeBackoff = currentCumulativeBackoff.plus(nextBackoffMillis);
  currentRetry += 1;
  return nextBackoffMillis;
}
 

@Override
public Row next() throws IOException {
  while (true) {
    try {
      Row result = currentDelegate.next();
      if (result != null) {
        lastRowKey = result.getKey();
      }
      // We've had at least one successful RPC, reset the backoff
      currentBackoff.reset();

      return result;
    } catch (IOExceptionWithStatus ioe) {
      Status.Code code = ioe.getStatus().getCode();
      if (code == Status.INTERNAL.getCode()
          || code == Status.UNAVAILABLE.getCode()
          || (retryOnDeadlineExceeded && code == Status.DEADLINE_EXCEEDED.getCode())) {
        long nextBackOff = currentBackoff.nextBackOffMillis();
        if (nextBackOff == BackOff.STOP) {
          throw new BigtableRetriesExhaustedException(
              "Exhausted streaming retries.", ioe);
        }
        sleep(nextBackOff);
        reissueRequest();
      } else {
        throw ioe;
      }
    }
  }
}
 

@Override
public long nextBackOffMillis() throws IOException {
  if (retriesAttempted >= maxRetries) {
    return BackOff.STOP;
  }
  long next = backoff.nextBackOffMillis();
  if (next == BackOff.STOP) {
    return BackOff.STOP;
  }
  retriesAttempted++;
  return next;
}
 

private static void upload(String classpathElement, String target, Path targetPath)
    throws IOException, InterruptedException {
  ArrayList<OpenOption> options = new ArrayList<>();
  options.add(WRITE);
  options.add(CREATE_NEW);
  if ("gs".equals(targetPath.toUri().getScheme())) {
    options.add(withMimeType(BINARY));
  }

  // Upload file, retrying on failure.
  Sleeper retrySleeper = Sleeper.DEFAULT;
  BackOff backoff = BACKOFF_FACTORY.backoff();
  while (true) {
    try {
      LOG.debug("Uploading classpath element {} to {}", classpathElement, target);
      try (WritableByteChannel writer = java.nio.file.Files.newByteChannel(targetPath,
          options.toArray(new OpenOption[options.size()]))) {
        copyContent(classpathElement, writer);
      }
      break;
    } catch (IOException e) {
      if (ERROR_EXTRACTOR.accessDenied(e)) {
        String errorMessage = String.format(
            "Uploaded failed due to permissions error, will NOT retry staging "
            + "of classpath %s. Please verify credentials are valid and that you have "
            + "write access to %s. Stale credentials can be resolved by executing "
            + "'gcloud auth login'.", classpathElement, target);
        LOG.error(errorMessage);
        throw new IOException(errorMessage, e);
      }
      long sleep = backoff.nextBackOffMillis();
      if (sleep == BackOff.STOP) {
        // Rethrow last error, to be included as a cause in the catch below.
        LOG.error("Upload failed, will NOT retry staging of classpath: {}",
            classpathElement, e);
        throw e;
      } else {
        LOG.warn("Upload attempt failed, sleeping before retrying staging of classpath: {}",
            classpathElement, e);
        retrySleeper.sleep(sleep);
      }
    }
  }
}
 

private StagingResult tryStagePackageWithRetry(
    PackageAttributes attributes, Sleeper retrySleeper, CreateOptions createOptions)
    throws IOException, InterruptedException {
  String sourceDescription = attributes.getSourceDescription();
  String target = attributes.getDestination().getLocation();
  BackOff backoff = BackOffAdapter.toGcpBackOff(BACKOFF_FACTORY.backoff());

  while (true) {
    try {
      return tryStagePackage(attributes, createOptions);
    } catch (IOException ioException) {

      if (ERROR_EXTRACTOR.accessDenied(ioException)) {
        String errorMessage =
            String.format(
                "Uploaded failed due to permissions error, will NOT retry staging "
                    + "of %s. Please verify credentials are valid and that you have "
                    + "write access to %s. Stale credentials can be resolved by executing "
                    + "'gcloud auth application-default login'.",
                sourceDescription, target);
        LOG.error(errorMessage);
        throw new IOException(errorMessage, ioException);
      }

      long sleep = backoff.nextBackOffMillis();
      if (sleep == BackOff.STOP) {
        LOG.error(
            "Upload failed, will NOT retry staging of package: {}",
            sourceDescription,
            ioException);
        throw new RuntimeException("Could not stage %s to %s", ioException);
      } else {
        LOG.warn(
            "Upload attempt failed, sleeping before retrying staging of package: {}",
            sourceDescription,
            ioException);
        retrySleeper.sleep(sleep);
      }
    }
  }
}
 

/**
 * Polls job until it is completed.
 *
 * @param bigquery the Bigquery instance to poll.
 * @param projectId the project that is polling.
 * @param jobReference the job to poll.
 * @param progressable to get progress of task.
 *
 * @throws IOException on IO Error.
 * @throws InterruptedException on sleep interrupt.
 */
public static void waitForJobCompletion(
    Bigquery bigquery,
    String projectId,
    JobReference jobReference,
    Progressable progressable)
    throws IOException, InterruptedException {

  Sleeper sleeper = Sleeper.DEFAULT;
  BackOff pollBackOff =
      new ExponentialBackOff.Builder()
          .setMaxIntervalMillis(POLL_WAIT_INTERVAL_MAX_MILLIS)
          .setInitialIntervalMillis(POLL_WAIT_INITIAL_MILLIS)
          .setMaxElapsedTimeMillis(POLL_WAIT_MAX_ELAPSED_MILLIS)
          .build();

  // Get starting time.
  long startTime = System.currentTimeMillis();
  long elapsedTime = 0;
  boolean notDone = true;

  // While job is incomplete continue to poll.
  while (notDone) {
    BackOff operationBackOff = new ExponentialBackOff.Builder().build();
    Get get = bigquery.jobs().get(projectId, jobReference.getJobId());
    
    Job pollJob = ResilientOperation.retry(
        ResilientOperation.getGoogleRequestCallable(get),
        operationBackOff,
        RetryDeterminer.RATE_LIMIT_ERRORS,
        IOException.class,
        sleeper);

    elapsedTime = System.currentTimeMillis() - startTime;
    LOG.debug("Job status ({} ms) {}: {}", elapsedTime, jobReference.getJobId(),
        pollJob.getStatus().getState());
    if (pollJob.getStatus().getState().equals("DONE")) {
      notDone = false;
      if (pollJob.getStatus().getErrorResult() != null) {
        throw new IOException(pollJob.getStatus().getErrorResult().getMessage());
      }
    } else {
      long millisToWait = pollBackOff.nextBackOffMillis();
      if (millisToWait == BackOff.STOP) {
        throw new IOException(
            String.format(
                "Job %s failed to complete after %s millis.",
                jobReference.getJobId(),
                elapsedTime));
      }
      // Pause execution for the configured duration before polling job status again.
      Thread.sleep(millisToWait);
      // Call progress to ensure task doesn't time out.
      progressable.progress();
    }
  }
}
 

/**
 * Helper to check whether an empty object already exists with the expected metadata specified
 * in {@code options}, to be used to determine whether it's safe to ignore an exception that
 * was thrown when trying to create the object, {@code exceptionOnCreate}.
 */
private boolean canIgnoreExceptionForEmptyObject(
    IOException exceptionOnCreate, StorageResourceId resourceId, CreateObjectOptions options)
    throws IOException {
  // TODO(user): Maybe also add 409 and even 412 errors if they pop up in this use case.
  // 500 ISE and 503 Service Unavailable tend to be raised when spamming GCS with create requests:
  if (errorExtractor.rateLimited(exceptionOnCreate)
      || errorExtractor.isInternalServerError(exceptionOnCreate)) {
    // We know that this is an error that is most often associated with trying to create an empty
    // object from multiple workers at the same time. We perform the following assuming that we
    // will eventually succeed and find an existing object. This will add up to a user-defined
    // maximum delay that caller will wait to receive an exception in the case of an incorrect
    // assumption and this being a scenario other than the multiple workers racing situation.
    GoogleCloudStorageItemInfo existingInfo;
    BackOff backOff;
    int maxWaitMillis = storageOptions.getMaxWaitMillisForEmptyObjectCreation();
    if (maxWaitMillis > 0) {
      backOff = new ExponentialBackOff.Builder()
          .setMaxElapsedTimeMillis(maxWaitMillis)
          .setMaxIntervalMillis(500)
          .setInitialIntervalMillis(100)
          .setMultiplier(1.5)
          .setRandomizationFactor(0.15)
          .build();
    } else {
      backOff = BackOff.STOP_BACKOFF;
    }
    long nextSleep = 0L;
    do {
      if (nextSleep > 0) {
        try {
          sleeper.sleep(nextSleep);
        } catch (InterruptedException e) {
          // We caught an InterruptedException, we should set the interrupted bit on this thread.
          Thread.currentThread().interrupt();
          nextSleep = BackOff.STOP;
        }
      }
      existingInfo = getItemInfo(resourceId);
      nextSleep = nextSleep == BackOff.STOP ? BackOff.STOP : backOff.nextBackOffMillis();
    } while (!existingInfo.exists() && nextSleep != BackOff.STOP);

    // Compare existence, size, and metadata; for 429 errors creating an empty object,
    // we don't care about metaGeneration/contentGeneration as long as the metadata
    // matches, since we don't know for sure whether our low-level request succeeded
    // first or some other client succeeded first.
    if (existingInfo.exists() && existingInfo.getSize() == 0) {
      if (!options.getRequireMetadataMatchForEmptyObjects()) {
        return true;
      } else if (existingInfo.metadataEquals(options.getMetadata())) {
        return true;
      }
    }
  }
  return false;
}