Java ExternalDataConfiguration類代碼示例

import com.google.api.services.bigquery.model.ExternalDataConfiguration; //導入依賴的package包/類
@Override
protected JobConfiguration jobConfiguration(String projectId)
{
    JobConfigurationQuery cfg = new JobConfigurationQuery()
            .setQuery(query);

    cfg.setUseLegacySql(params.get("use_legacy_sql", boolean.class, false));

    params.getOptional("allow_large_results", boolean.class).transform(cfg::setAllowLargeResults);
    params.getOptional("use_query_cache", Boolean.class).transform(cfg::setUseQueryCache);
    params.getOptional("create_disposition", String.class).transform(cfg::setCreateDisposition);
    params.getOptional("write_disposition", String.class).transform(cfg::setWriteDisposition);
    params.getOptional("flatten_results", Boolean.class).transform(cfg::setFlattenResults);
    params.getOptional("maximum_billing_tier", Integer.class).transform(cfg::setMaximumBillingTier);
    params.getOptional("priority", String.class).transform(cfg::setPriority);

    params.getOptional("table_definitions", new TypeReference<Map<String, ExternalDataConfiguration>>() {})
            .transform(cfg::setTableDefinitions);
    params.getOptional("user_defined_function_resources", new TypeReference<List<UserDefinedFunctionResource>>() {})
            .transform(cfg::setUserDefinedFunctionResources);

    Optional<DatasetReference> defaultDataset = params.getOptional("dataset", String.class)
            .transform(Bq::datasetReference);
    defaultDataset.transform(cfg::setDefaultDataset);

    params.getOptional("destination_table", String.class)
            .transform(s -> cfg.setDestinationTable(tableReference(projectId, defaultDataset, s)));

    return new JobConfiguration()
            .setQuery(cfg);
}
 

import com.google.api.services.bigquery.model.ExternalDataConfiguration; //導入依賴的package包/類
/**
 * Performs a federated import on data from GCS into BigQuery via a table insert.
 *
 * @param projectId the project on whose behalf to perform the load.
 * @param tableRef the reference to the destination table.
 * @param schema the schema of the source data to populate the destination table by.
 * @param sourceFormat the file format of the source data.
 * @param gcsPaths the location of the source data in GCS.
 * @throws IOException
 */
public void importFederatedFromGcs(
    String projectId,
    TableReference tableRef,
    @Nullable TableSchema schema,
    BigQueryFileFormat sourceFormat,
    List<String> gcsPaths)
    throws IOException {
  LOG.info(
      "Importing into federated table '{}' from {} paths; path[0] is '{}'",
      BigQueryStrings.toString(tableRef),
      gcsPaths.size(),
      gcsPaths.isEmpty() ? "(empty)" : gcsPaths.get(0));

  ExternalDataConfiguration externalConf = new ExternalDataConfiguration();
  externalConf.setSchema(schema);
  externalConf.setSourceUris(gcsPaths);
  externalConf.setSourceFormat(sourceFormat.getFormatIdentifier());

  // Auto detect the schema if we're not given one, otherwise use the passed schema.
  if (schema == null) {
    LOG.info("No federated import schema provided, auto detecting schema.");
    externalConf.setAutodetect(true);
  } else {
    LOG.info("Using provided federated import schema '{}'.", schema.toString());
  }

  Table table = new Table();
  table.setTableReference(tableRef);
  table.setExternalDataConfiguration(externalConf);

  service.tables().insert(projectId, tableRef.getDatasetId(), table).execute();
}
 

import com.google.api.services.bigquery.model.ExternalDataConfiguration; //導入依賴的package包/類
@Before
public void setup() {
  conf = new Configuration();
  table = new Table().setExternalDataConfiguration(
      new ExternalDataConfiguration()
          .setSourceFormat("AVRO")
          .setSourceUris(
              ImmutableList.of(
                  "gs://foo-bucket/bar-dir/glob-*.avro",
                  "gs://foo-bucket/bar-dir/file.avro")));
}
 

import com.google.api.services.bigquery.model.ExternalDataConfiguration; //導入依賴的package包/類
/**
 * Tests getSplits method of GsonBigQueryInputFormat with federated data.
 */
@Test
public void testGetSplitsFederated()
    throws IOException, InterruptedException {
  BigQueryJobWrapper wrapper = new BigQueryJobWrapper(config);
  wrapper.setJobID(new JobID());
  config.unset(BigQueryConfiguration.INPUT_QUERY_KEY);

  table.setType("EXTERNAL")
      .setExternalDataConfiguration(
          new ExternalDataConfiguration()
              .setSourceFormat("NEWLINE_DELIMITED_JSON")
              .setSourceUris(ImmutableList.of("gs://foo-bucket/bar.json")));

  FileSplit split = new FileSplit(new Path("gs://foo-bucket/bar.json"), 0, 100, new String[0]);
  when(mockInputFormat.getSplits(eq(wrapper))).thenReturn(ImmutableList.<InputSplit>of(split));

  GsonBigQueryInputFormat gsonBigQueryInputFormat = new GsonBigQueryInputFormatForTest();
  gsonBigQueryInputFormat.setDelegateInputFormat(mockInputFormat);

  // Run getSplits method.
  List<InputSplit> splits = gsonBigQueryInputFormat.getSplits(wrapper);

  assertEquals(1, splits.size());
  assertEquals(split.getPath(), ((FileSplit) splits.get(0)).getPath());
  assertEquals("gs://foo-bucket/bar.json", config.get("mapred.input.dir"));
  verify(mockBigQueryHelper, times(1)).getTable(eq(tableRef));
  verifyNoMoreInteractions(mockBigquery);
}