Java Job.setConfiguration方法代码示例

import com.google.api.services.bigquery.model.Job; //导入方法依赖的package包/类
@Override
public void startExtractJob(JobReference jobRef, JobConfigurationExtract extractConfig)
    throws InterruptedException, IOException {
  checkArgument(extractConfig.getDestinationFormat().equals("AVRO"),
      "Only extract to AVRO is supported");
  synchronized (allJobs) {
    verifyUniqueJobId(jobRef.getJobId());
    ++numExtractJobCalls;

    Job job = new Job();
    job.setJobReference(jobRef);
    job.setConfiguration(new JobConfiguration().setExtract(extractConfig));
    job.setKind(" bigquery#job");
    job.setStatus(new JobStatus().setState("PENDING"));
    allJobs.put(jobRef.getProjectId(), jobRef.getJobId(), new JobInfo(job));
  }
}
 

import com.google.api.services.bigquery.model.Job; //导入方法依赖的package包/类
/**
 * Helper method to get the load request to BigQuery with numValues copies of jsonValue.
 */
private Job getExpectedJob() {
  // Configure a write job.
  JobConfigurationLoad loadConfig = new JobConfigurationLoad();
  loadConfig.setCreateDisposition("CREATE_IF_NEEDED");
  loadConfig.setWriteDisposition("WRITE_TRUNCATE");
  loadConfig.setSourceFormat("NEWLINE_DELIMITED_JSON");

  // Describe the resulting table you are importing to:
  loadConfig.setDestinationTable(getSampleTableRef());

  // Create and set the output schema.
  TableSchema schema = new TableSchema();
  schema.setFields(fields);
  loadConfig.setSchema(schema);

  // Create Job configuration.
  JobConfiguration jobConfig = new JobConfiguration();
  jobConfig.setLoad(loadConfig);

  // Set the output write job.
  Job expectedJob = new Job();
  expectedJob.setConfiguration(jobConfig);
  expectedJob.setJobReference(jobReference);
  return expectedJob;
}
 

import com.google.api.services.bigquery.model.Job; //导入方法依赖的package包/类
@Override
public void startLoadJob(JobReference jobRef, JobConfigurationLoad loadConfig)
    throws InterruptedException, IOException {
  synchronized (allJobs) {
    verifyUniqueJobId(jobRef.getJobId());
    Job job = new Job();
    job.setJobReference(jobRef);
    job.setConfiguration(new JobConfiguration().setLoad(loadConfig));
    job.setKind(" bigquery#job");
    job.setStatus(new JobStatus().setState("PENDING"));

    // Copy the files to a new location for import, as the temporary files will be deleted by
    // the caller.
    if (loadConfig.getSourceUris().size() > 0) {
      ImmutableList.Builder<ResourceId> sourceFiles = ImmutableList.builder();
      ImmutableList.Builder<ResourceId> loadFiles = ImmutableList.builder();
      for (String filename : loadConfig.getSourceUris()) {
        sourceFiles.add(FileSystems.matchNewResource(filename, false /* isDirectory */));
        loadFiles.add(FileSystems.matchNewResource(
            filename + ThreadLocalRandom.current().nextInt(), false /* isDirectory */));
      }

      FileSystems.copy(sourceFiles.build(), loadFiles.build());
      filesForLoadJobs.put(jobRef.getProjectId(), jobRef.getJobId(), loadFiles.build());
    }

    allJobs.put(jobRef.getProjectId(), jobRef.getJobId(), new JobInfo(job));
  }
}
 

import com.google.api.services.bigquery.model.Job; //导入方法依赖的package包/类
@Override
public void startQueryJob(JobReference jobRef, JobConfigurationQuery query)
    throws IOException, InterruptedException {
  synchronized (allJobs) {
    Job job = new Job();
    job.setJobReference(jobRef);
    job.setConfiguration(new JobConfiguration().setQuery(query));
    job.setKind(" bigquery#job");
    job.setStatus(new JobStatus().setState("PENDING"));
    allJobs.put(jobRef.getProjectId(), jobRef.getJobId(), new JobInfo(job));
  }
}
 

import com.google.api.services.bigquery.model.Job; //导入方法依赖的package包/类
@Override
public void startCopyJob(JobReference jobRef, JobConfigurationTableCopy copyConfig)
    throws IOException, InterruptedException {
  synchronized (allJobs) {
    verifyUniqueJobId(jobRef.getJobId());
    Job job = new Job();
    job.setJobReference(jobRef);
    job.setConfiguration(new JobConfiguration().setCopy(copyConfig));
    job.setKind(" bigquery#job");
    job.setStatus(new JobStatus().setState("PENDING"));
    allJobs.put(jobRef.getProjectId(), jobRef.getJobId(), new JobInfo(job));
  }
}
 

import com.google.api.services.bigquery.model.Job; //导入方法依赖的package包/类
@Override
public void beginExport() throws IOException {
  // Create job and configuration.
  JobConfigurationExtract extractConfig = new JobConfigurationExtract();

  // Set source.
  extractConfig.setSourceTable(tableToExport.getTableReference());

  // Set destination.
  extractConfig.setDestinationUris(getExportPaths());
  extractConfig.set(DESTINATION_FORMAT_KEY, fileFormat.getFormatIdentifier());

  JobConfiguration config = new JobConfiguration();
  config.setExtract(extractConfig);

  JobReference jobReference =
      bigQueryHelper.createJobReference(projectId, "exporttocloudstorage");

  Job job = new Job();
  job.setConfiguration(config);
  job.setJobReference(jobReference);

  // Insert and run job.
  try {
    Job response = bigQueryHelper.insertJobOrFetchDuplicate(projectId, job);
    LOG.debug("Got response '{}'", response);
    exportJobReference = response.getJobReference();
  } catch (IOException e) {
    String error = String.format(
        "Error while exporting table %s",
        BigQueryStrings.toString(tableToExport.getTableReference()));
    LOG.error(error, e);
    throw new IOException(error, e);
  }
}
 

import com.google.api.services.bigquery.model.Job; //导入方法依赖的package包/类
/**
 * Runs the query in BigQuery and writes results to a temporary table.
 *
 * @param bigquery the Bigquery instance to use.
 * @param projectId the project on whose behalf the query will be run.
 * @param tableRef the table to write the results to.
 * @param query the query to run.
 * @throws IOException on IO error.
 * @throws InterruptedException on interrupt.
 */
@VisibleForTesting
static void runQuery(
    BigQueryHelper bigQueryHelper, String projectId, TableReference tableRef, String query)
    throws IOException, InterruptedException {
  LOG.debug("runQuery(bigquery, '{}', '{}', '{}')",
      projectId, BigQueryStrings.toString(tableRef), query);

  // Create a query statement and query request object.
  JobConfigurationQuery queryConfig = new JobConfigurationQuery();
  queryConfig.setAllowLargeResults(true);
  queryConfig.setQuery(query);

  // Set the table to put results into.
  queryConfig.setDestinationTable(tableRef);

  // Require table to be empty.
  queryConfig.setWriteDisposition("WRITE_EMPTY");

  JobConfiguration config = new JobConfiguration();
  config.setQuery(queryConfig);

  JobReference jobReference = bigQueryHelper.createJobReference(projectId, "querybasedexport");

  Job job = new Job();
  job.setConfiguration(config);
  job.setJobReference(jobReference);

  // Run the job.
  Job response = bigQueryHelper.insertJobOrFetchDuplicate(projectId, job);
  LOG.debug("Got response '{}'", response);

  // Create anonymous Progressable object
  Progressable progressable = new Progressable() {
    @Override
    public void progress() {
      // TODO(user): ensure task doesn't time out
    }
  };

  // Poll until job is complete.
  BigQueryUtils.waitForJobCompletion(
      bigQueryHelper.getRawBigquery(), projectId, jobReference, progressable);
}
 

import com.google.api.services.bigquery.model.Job; //导入方法依赖的package包/类
/**
 * Imports data from GCS into BigQuery via a load job. Optionally polls for completion before
 * returning.
 *
 * @param projectId the project on whose behalf to perform the load.
 * @param tableRef the reference to the destination table.
 * @param schema the schema of the source data to populate the destination table by.
 * @param sourceFormat the file format of the source data.
 * @param writeDisposition the write disposition of the output table.
 * @param gcsPaths the location of the source data in GCS.
 * @param awaitCompletion if true, block and poll until job completes, otherwise return as soon as
 *     the job has been successfully dispatched.
 * @throws IOException
 * @throws InterruptedException if interrupted while waiting for job completion.
 */
public void importFromGcs(
    String projectId,
    TableReference tableRef,
    @Nullable TableSchema schema,
    BigQueryFileFormat sourceFormat,
    String writeDisposition,
    List<String> gcsPaths,
    boolean awaitCompletion)
    throws IOException, InterruptedException {
  LOG.info(
      "Importing into table '{}' from {} paths; path[0] is '{}'; awaitCompletion: {}",
      BigQueryStrings.toString(tableRef),
      gcsPaths.size(),
      gcsPaths.isEmpty() ? "(empty)" : gcsPaths.get(0),
      awaitCompletion);

  // Create load conf with minimal requirements.
  JobConfigurationLoad loadConfig = new JobConfigurationLoad();
  loadConfig.setSchema(schema);
  loadConfig.setSourceFormat(sourceFormat.getFormatIdentifier());
  loadConfig.setSourceUris(gcsPaths);
  loadConfig.setDestinationTable(tableRef);
  loadConfig.setWriteDisposition(writeDisposition);

  // Auto detect the schema if we're not given one, otherwise use the passed schema.
  if (schema == null) {
    LOG.info("No import schema provided, auto detecting schema.");
    loadConfig.setAutodetect(true);
  } else {
    LOG.info("Using provided import schema '{}'.", schema.toString());
  }

  JobConfiguration config = new JobConfiguration();
  config.setLoad(loadConfig);

  JobReference jobReference = createJobReference(projectId, "direct-bigqueryhelper-import");
  Job job = new Job();
  job.setConfiguration(config);
  job.setJobReference(jobReference);

  // Insert and run job.
  insertJobOrFetchDuplicate(projectId, job);

  if (awaitCompletion) {
    // Poll until job is complete.
    BigQueryUtils.waitForJobCompletion(
        getRawBigquery(), projectId, jobReference, NOP_PROGRESSABLE);
  }
}
 

import com.google.api.services.bigquery.model.Job; //导入方法依赖的package包/类
/**
 * Exports BigQuery results into GCS, polls for completion before returning.
 *
 * @param projectId the project on whose behalf to perform the export.
 * @param tableRef the table to export.
 * @param gcsPaths the GCS paths to export to.
 * @param awaitCompletion if true, block and poll until job completes, otherwise return as soon as
 *        the job has been successfully dispatched.
 *
 * @throws IOException on IO error.
 * @throws InterruptedException on interrupt.
 */
public void exportBigQueryToGcs(String projectId, TableReference tableRef, List<String> gcsPaths,
    boolean awaitCompletion) throws IOException, InterruptedException {
  LOG.debug(
      "exportBigQueryToGcs(bigquery, '{}', '{}', '{}', '{}')",
      projectId,
      BigQueryStrings.toString(tableRef),
      gcsPaths,
      awaitCompletion);
  LOG.info(
      "Exporting table '{}' to {} paths; path[0] is '{}'; awaitCompletion: {}",
      BigQueryStrings.toString(tableRef),
      gcsPaths.size(),
      gcsPaths.isEmpty() ? "(empty)" : gcsPaths.get(0),
      awaitCompletion);

  // Create job and configuration.
  JobConfigurationExtract extractConfig = new JobConfigurationExtract();

  // Set source.
  extractConfig.setSourceTable(tableRef);

  // Set destination.
  extractConfig.setDestinationUris(gcsPaths);
  extractConfig.set("destinationFormat", "NEWLINE_DELIMITED_JSON");

  JobConfiguration config = new JobConfiguration();
  config.setExtract(extractConfig);

  JobReference jobReference = createJobReference(projectId, "direct-bigqueryhelper-export");

  Job job = new Job();
  job.setConfiguration(config);
  job.setJobReference(jobReference);

  // Insert and run job.
  insertJobOrFetchDuplicate(projectId, job);

  if (awaitCompletion) {
    // Poll until job is complete.
    BigQueryUtils.waitForJobCompletion(service, projectId, jobReference, NOP_PROGRESSABLE);
  }
}
 

import com.google.api.services.bigquery.model.Job; //导入方法依赖的package包/类
/**
 * Moves the files from the working dataset to the job output table.
 *
 * @param context the task context.
 * @throws IOException on IO Error.
 */
@Override
public void commitTask(TaskAttemptContext context) throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug("commitTask({})", HadoopToStringUtil.toString(context));
  }

  // Create a table copy request object.
  JobConfigurationTableCopy copyTableConfig = new JobConfigurationTableCopy();

  // Set the table to get results from.
  copyTableConfig.setSourceTable(tempTableRef);

  // Set the table to put results into.
  copyTableConfig.setDestinationTable(finalTableRef);

  copyTableConfig.setWriteDisposition("WRITE_APPEND");

  JobConfiguration config = new JobConfiguration();
  config.setCopy(copyTableConfig);

  JobReference jobReference = bigQueryHelper.createJobReference(
      projectId, context.getTaskAttemptID().toString());

  Job job = new Job();
  job.setConfiguration(config);
  job.setJobReference(jobReference);

  // Run the job.
  LOG.debug("commitTask: Running table copy from {} to {}",
      BigQueryStrings.toString(tempTableRef), BigQueryStrings.toString(finalTableRef));
  Job response = bigQueryHelper.insertJobOrFetchDuplicate(projectId, job);
  LOG.debug("Got response '{}'", response);

  // Poll until job is complete.
  try {
    BigQueryUtils.waitForJobCompletion(
        bigQueryHelper.getRawBigquery(), projectId, jobReference, context);
  } catch (InterruptedException e) {
    LOG.error("Could not check if results of task were transfered.", e);
    throw new IOException("Could not check if results of task were transfered.", e);
  }
  LOG.info("Saved output of task to table '{}' using project '{}'",
      BigQueryStrings.toString(finalTableRef), projectId);
}
 

import com.google.api.services.bigquery.model.Job; //导入方法依赖的package包/类
/**
 * Runs the BigQuery Job for getting the data and waits for completion.
 * 
 * @return
 * @throws IOException
 * @throws InterruptedException
 */
private void executeJob(@Nullable final JobReference activeJob) throws IOException, InterruptedException {
	logger.debug("Running Job to fetch data from BigQuery");
	JobReference jobReference;
	if (activeJob == null) {
		fetchedRows = new BigInteger("0");
		final Job job = new Job();
		final JobConfiguration config = new JobConfiguration();
		final JobConfigurationQuery queryConfig = new JobConfigurationQuery();

		job.setConfiguration(config.setQuery(queryConfig.setQuery(evalJavaScript(query))));

		final Insert insert = client.jobs().insert(project, job).setProjectId(project);
		jobReference = insert.execute().getJobReference();
	}
	else {
		jobReference = activeJob;
	}

	while (!stopThread) {
		final Job pollJob = client.jobs().get(project, jobReference.getJobId()).execute();

		if (pollJob.getStatus().getState().equals("DONE")) {
			logger.debug("BigQuery Job has finished, fetching results");

			if (pollJob.getStatus().getErrorResult() != null) {
				logger.info("BigQuery reported an error for the query: {}", pollJob.getStatus().getErrorResult().getMessage());
				return;
			}

			final GetQueryResultsResponse queryResult = client.jobs()
				.getQueryResults(project, pollJob.getJobReference().getJobId())
				.execute();

			if (queryResult.getTotalRows().equals(BigInteger.ZERO)) {
				logger.info("Got 0 results from BigQuery - not gonna do anything");
				return;
			}

			logger.trace("Getting column names from BigQuery to be used for building json structure");
			columns = new ArrayList<>();
			for (TableFieldSchema fieldSchema : queryResult.getSchema().getFields()) {
				columns.add(fieldSchema.getName());
			}
			fetchedRows = fetchedRows.add(new BigInteger("" + queryResult.getRows().size()));

			parse(queryResult.getRows());

			if (fetchedRows.compareTo(queryResult.getTotalRows()) < 0) {
				logger.debug("Continuing BigQuery job as not all rows could be fetched on the first request");
				executeJob(jobReference);
			}
			return;
		}
		try {
			logger.trace("Waiting for BigQuery job to be done (state is {})", pollJob.getStatus().getState());
			Thread.sleep(1000);
		} catch (InterruptedException e) {
			logger.trace("Unable to put thread to sleep", e);
		}
	}
}