Java JobConfigurationTableCopy类代码示例

import com.google.api.services.bigquery.model.JobConfigurationTableCopy; //导入依赖的package包/类
/**
 * {@inheritDoc}
 *
 * <p>Tries executing the RPC for at most {@code MAX_RPC_RETRIES} times until it succeeds.
 *
 * @throws IOException if it exceeds {@code MAX_RPC_RETRIES} attempts.
 */
@Override
public void startCopyJob(JobReference jobRef, JobConfigurationTableCopy copyConfig)
    throws IOException, InterruptedException {
  Job job = new Job()
      .setJobReference(jobRef)
      .setConfiguration(
          new JobConfiguration().setCopy(copyConfig));

  startJob(job, errorExtractor, client);
}
 

import com.google.api.services.bigquery.model.JobConfigurationTableCopy; //导入依赖的package包/类
@Override
public void startCopyJob(JobReference jobRef, JobConfigurationTableCopy copyConfig)
    throws IOException, InterruptedException {
  synchronized (allJobs) {
    verifyUniqueJobId(jobRef.getJobId());
    Job job = new Job();
    job.setJobReference(jobRef);
    job.setConfiguration(new JobConfiguration().setCopy(copyConfig));
    job.setKind(" bigquery#job");
    job.setStatus(new JobStatus().setState("PENDING"));
    allJobs.put(jobRef.getProjectId(), jobRef.getJobId(), new JobInfo(job));
  }
}
 

import com.google.api.services.bigquery.model.JobConfigurationTableCopy; //导入依赖的package包/类
private JobStatus runCopyJob(JobConfigurationTableCopy copy)
    throws InterruptedException, IOException {
  List<TableReference> sources = copy.getSourceTables();
  TableReference destination = copy.getDestinationTable();
  WriteDisposition writeDisposition = WriteDisposition.valueOf(copy.getWriteDisposition());
  CreateDisposition createDisposition = CreateDisposition.valueOf(copy.getCreateDisposition());
  Table existingTable = datasetService.getTable(destination);
  if (!validateDispositions(existingTable, createDisposition, writeDisposition)) {
    return new JobStatus().setState("FAILED").setErrorResult(new ErrorProto());
  }
  TimePartitioning partitioning = null;
  TableSchema schema = null;
  boolean first = true;
  List<TableRow> allRows = Lists.newArrayList();
  for (TableReference source : sources) {
    Table table = checkNotNull(datasetService.getTable(source));
    if (!first) {
      if (partitioning != table.getTimePartitioning()) {
        return new JobStatus().setState("FAILED").setErrorResult(new ErrorProto());
      }
      if (schema != table.getSchema()) {
        return new JobStatus().setState("FAILED").setErrorResult(new ErrorProto());
      }
    }
    partitioning = table.getTimePartitioning();
    schema = table.getSchema();
    first = false;
    allRows.addAll(datasetService.getAllRows(
        source.getProjectId(), source.getDatasetId(), source.getTableId()));
  }
  datasetService.createTable(new Table()
      .setTableReference(destination)
      .setSchema(schema)
      .setTimePartitioning(partitioning));
  datasetService.insertAll(destination, allRows, null);
  return new JobStatus().setState("DONE");
}
 

import com.google.api.services.bigquery.model.JobConfigurationTableCopy; //导入依赖的package包/类
private void copy(
    JobService jobService,
    DatasetService datasetService,
    String jobIdPrefix,
    TableReference ref,
    List<TableReference> tempTables,
    WriteDisposition writeDisposition,
    CreateDisposition createDisposition,
    @Nullable String tableDescription) throws InterruptedException, IOException {
  JobConfigurationTableCopy copyConfig = new JobConfigurationTableCopy()
      .setSourceTables(tempTables)
      .setDestinationTable(ref)
      .setWriteDisposition(writeDisposition.name())
      .setCreateDisposition(createDisposition.name());

  String projectId = ref.getProjectId();
  Job lastFailedCopyJob = null;
  for (int i = 0; i < BatchLoads.MAX_RETRY_JOBS; ++i) {
    String jobId = jobIdPrefix + "-" + i;
    JobReference jobRef = new JobReference()
        .setProjectId(projectId)
        .setJobId(jobId);
    jobService.startCopyJob(jobRef, copyConfig);
    Job copyJob = jobService.pollJob(jobRef, BatchLoads.LOAD_JOB_POLL_MAX_RETRIES);
    Status jobStatus = BigQueryHelpers.parseStatus(copyJob);
    switch (jobStatus) {
      case SUCCEEDED:
        if (tableDescription != null) {
          datasetService.patchTableDescription(ref, tableDescription);
        }
        return;
      case UNKNOWN:
        throw new RuntimeException(String.format(
            "UNKNOWN status of copy job [%s]: %s.", jobId,
            BigQueryHelpers.jobToPrettyString(copyJob)));
      case FAILED:
        lastFailedCopyJob = copyJob;
        continue;
      default:
        throw new IllegalStateException(String.format(
            "Unexpected status [%s] of load job: %s.",
            jobStatus, BigQueryHelpers.jobToPrettyString(copyJob)));
    }
  }
  throw new RuntimeException(String.format(
      "Failed to create copy job with id prefix %s, "
          + "reached max retries: %d, last failed copy job: %s.",
      jobIdPrefix,
      BatchLoads.MAX_RETRY_JOBS,
      BigQueryHelpers.jobToPrettyString(lastFailedCopyJob)));
}
 

import com.google.api.services.bigquery.model.JobConfigurationTableCopy; //导入依赖的package包/类
/**
 * Moves the files from the working dataset to the job output table.
 *
 * @param context the task context.
 * @throws IOException on IO Error.
 */
@Override
public void commitTask(TaskAttemptContext context) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("commitTask({})", HadoopToStringUtil.toString(context));
  }

  // Create a table copy request object.
  JobConfigurationTableCopy copyTableConfig = new JobConfigurationTableCopy();

  // Set the table to get results from.
  copyTableConfig.setSourceTable(tempTableRef);

  // Set the table to put results into.
  copyTableConfig.setDestinationTable(finalTableRef);

  copyTableConfig.setWriteDisposition("WRITE_APPEND");

  JobConfiguration config = new JobConfiguration();
  config.setCopy(copyTableConfig);

  JobReference jobReference = bigQueryHelper.createJobReference(
      projectId, context.getTaskAttemptID().toString());

  Job job = new Job();
  job.setConfiguration(config);
  job.setJobReference(jobReference);

  // Run the job.
  LOG.debug("commitTask: Running table copy from {} to {}",
      BigQueryStrings.toString(tempTableRef), BigQueryStrings.toString(finalTableRef));
  Job response = bigQueryHelper.insertJobOrFetchDuplicate(projectId, job);
  LOG.debug("Got response '{}'", response);

  // Poll until job is complete.
  try {
    BigQueryUtils.waitForJobCompletion(
        bigQueryHelper.getRawBigquery(), projectId, jobReference, context);
  } catch (InterruptedException e) {
    LOG.error("Could not check if results of task were transfered.", e);
    throw new IOException("Could not check if results of task were transfered.", e);
  }
  LOG.info("Saved output of task to table '{}' using project '{}'",
      BigQueryStrings.toString(finalTableRef), projectId);
}
 

import com.google.api.services.bigquery.model.JobConfigurationTableCopy; //导入依赖的package包/类
/**
 * Start a BigQuery copy job.
 */
void startCopyJob(JobReference jobRef, JobConfigurationTableCopy copyConfig)
    throws IOException, InterruptedException;