Java TableReference类代码示例

import com.google.api.services.bigquery.model.TableReference; //导入依赖的package包/类
/**
 * input - a tupel that contains the data element (TableRow), the window, the timestamp, and the pane
 */

@Override
public TableDestination apply(ValueInSingleWindow<TableRow> input) {
    
	String partition;
	
	if (this.isTimeField) {
     String sTime = (String) input.getValue().get(this.fieldName);
     Instant time = Instant.parse(sTime);
     partition = time.toString(partitionFormatter);
	} else {
		partition = ((Integer) input.getValue().get(this.fieldName)).toString();
	}
	
    TableReference reference = new TableReference();
    reference.setProjectId(this.projectId);
    reference.setDatasetId(this.datasetId);
    reference.setTableId(this.partitionPrefix + partition);
    return new TableDestination(reference, null);
}
 

import com.google.api.services.bigquery.model.TableReference; //导入依赖的package包/类
/**
 * Run a batch pipeline.
 */
public static void main(String[] args) throws Exception {
  Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
  Pipeline pipeline = Pipeline.create(options);

  TableReference tableRef = new TableReference();
  tableRef.setDatasetId(options.as(Options.class).getOutputDataset());
  tableRef.setProjectId(options.as(GcpOptions.class).getProject());
  tableRef.setTableId(options.getOutputTableName());

  // Read events from a CSV file and parse them.
  pipeline
      .apply(TextIO.Read.from(options.getInput()))
      .apply(ParDo.named("ParseGameEvent").of(new ParseEventFn()))
      // Extract and sum username/score pairs from the event data.
      .apply("ExtractUserScore", new ExtractAndSumScore("user"))
      // Write the results to BigQuery.
      .apply(ParDo.named("FormatUserScoreSums").of(new FormatUserScoreSumsFn()))
      .apply(
          BigQueryIO.Write.to(tableRef)
              .withSchema(FormatUserScoreSumsFn.getSchema())
              .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
              .withWriteDisposition(WriteDisposition.WRITE_APPEND));

  pipeline.run();
}
 

import com.google.api.services.bigquery.model.TableReference; //导入依赖的package包/类
/** Run a batch or streaming pipeline. */
public static void main(String[] args) throws Exception {
  Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);

  Pipeline pipeline = Pipeline.create(options);

  TableReference tableRef = new TableReference();
  tableRef.setDatasetId(options.as(Options.class).getOutputDataset());
  tableRef.setProjectId(options.as(GcpOptions.class).getProject());
  tableRef.setTableId(options.getOutputTableName());

  // Read events from either a CSV file or PubSub stream.
  pipeline
      .apply(new ReadGameEvents(options))
      .apply("WindowedTeamScore", new Exercise2.WindowedTeamScore(Duration.standardMinutes(60)))
      // Write the results to BigQuery.
      .apply(ParDo.named("FormatTeamScoreSums").of(new Exercise2.FormatTeamScoreSumsFn()))
      .apply(
          BigQueryIO.Write.to(tableRef)
              .withSchema(Exercise2.FormatTeamScoreSumsFn.getSchema())
              .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
              .withWriteDisposition(WriteDisposition.WRITE_APPEND));

  pipeline.run();
}
 

import com.google.api.services.bigquery.model.TableReference; //导入依赖的package包/类
/**
 * Run a batch pipeline.
 */
public static void main(String[] args) throws Exception {
  Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
  Pipeline pipeline = Pipeline.create(options);

  TableReference tableRef = new TableReference();
  tableRef.setDatasetId(options.as(Options.class).getOutputDataset());
  tableRef.setProjectId(options.as(GcpOptions.class).getProject());
  tableRef.setTableId(options.getOutputTableName());

  // Read events from a CSV file, parse them and write (import) them to BigQuery.
  pipeline
      .apply(TextIO.Read.from(options.getInput()))
      .apply(ParDo.named("ParseGameEvent").of(new ParseEventFn()))
      .apply(ParDo.named("FormatGameEvent").of(new FormatGameEventFn()))
      .apply(
          BigQueryIO.Write.to(tableRef)
              .withSchema(FormatGameEventFn.getSchema())
              .withCreateDisposition(CreateDisposition.CREATE_IF_NEEDED)
              .withWriteDisposition(WriteDisposition.WRITE_APPEND));

  pipeline.run();
}
 

import com.google.api.services.bigquery.model.TableReference; //导入依赖的package包/类
private void setupBigQueryTable(String projectId, String datasetId, String tableId,
    TableSchema schema) throws IOException {
  if (bigQueryClient == null) {
    bigQueryClient = Transport.newBigQueryClient(options.as(BigQueryOptions.class)).build();
  }

  Datasets datasetService = bigQueryClient.datasets();
  if (executeNullIfNotFound(datasetService.get(projectId, datasetId)) == null) {
    Dataset newDataset = new Dataset().setDatasetReference(
        new DatasetReference().setProjectId(projectId).setDatasetId(datasetId));
    datasetService.insert(projectId, newDataset).execute();
  }

  Tables tableService = bigQueryClient.tables();
  Table table = executeNullIfNotFound(tableService.get(projectId, datasetId, tableId));
  if (table == null) {
    Table newTable = new Table().setSchema(schema).setTableReference(
        new TableReference().setProjectId(projectId).setDatasetId(datasetId).setTableId(tableId));
    tableService.insert(projectId, datasetId, newTable).execute();
  } else if (!table.getSchema().equals(schema)) {
    throw new RuntimeException(
        "Table exists and schemas do not match, expecting: " + schema.toPrettyString()
        + ", actual: " + table.getSchema().toPrettyString());
  }
}
 

import com.google.api.services.bigquery.model.TableReference; //导入依赖的package包/类
private void setupBigQueryTable(String projectId, String datasetId, String tableId,
    TableSchema schema) throws IOException {
  if (bigQueryClient == null) {
    bigQueryClient = newBigQueryClient(options.as(BigQueryOptions.class)).build();
  }

  Datasets datasetService = bigQueryClient.datasets();
  if (executeNullIfNotFound(datasetService.get(projectId, datasetId)) == null) {
    Dataset newDataset = new Dataset().setDatasetReference(
        new DatasetReference().setProjectId(projectId).setDatasetId(datasetId));
    datasetService.insert(projectId, newDataset).execute();
  }

  Tables tableService = bigQueryClient.tables();
  Table table = executeNullIfNotFound(tableService.get(projectId, datasetId, tableId));
  if (table == null) {
    Table newTable = new Table().setSchema(schema).setTableReference(
        new TableReference().setProjectId(projectId).setDatasetId(datasetId).setTableId(tableId));
    tableService.insert(projectId, datasetId, newTable).execute();
  } else if (!table.getSchema().equals(schema)) {
    throw new RuntimeException(
        "Table exists and schemas do not match, expecting: " + schema.toPrettyString()
        + ", actual: " + table.getSchema().toPrettyString());
  }
}
 

import com.google.api.services.bigquery.model.TableReference; //导入依赖的package包/类
static void verifyTableNotExistOrEmpty(DatasetService datasetService, TableReference tableRef) {
  try {
    if (datasetService.getTable(tableRef) != null) {
      checkState(
          datasetService.isTableEmpty(tableRef),
          "BigQuery table is not empty: %s.",
          toTableSpec(tableRef));
    }
  } catch (IOException | InterruptedException e) {
    if (e instanceof InterruptedException) {
      Thread.currentThread().interrupt();
    }
    throw new RuntimeException(
        "unable to confirm BigQuery table emptiness for table " + toTableSpec(tableRef), e);
  }
}
 

import com.google.api.services.bigquery.model.TableReference; //导入依赖的package包/类
static void verifyDatasetPresence(DatasetService datasetService, TableReference table) {
  try {
    datasetService.getDataset(table.getProjectId(), table.getDatasetId());
  } catch (Exception e) {
    ApiErrorExtractor errorExtractor = new ApiErrorExtractor();
    if ((e instanceof IOException) && errorExtractor.itemNotFound((IOException) e)) {
      throw new IllegalArgumentException(
          String.format(RESOURCE_NOT_FOUND_ERROR, "dataset", toTableSpec(table)), e);
    } else if (e instanceof RuntimeException) {
      throw (RuntimeException) e;
    } else {
      throw new RuntimeException(
          String.format(
              UNABLE_TO_CONFIRM_PRESENCE_OF_RESOURCE_ERROR, "dataset", toTableSpec(table)),
          e);
    }
  }
}
 

import com.google.api.services.bigquery.model.TableReference; //导入依赖的package包/类
@Test
public void testRemoveTemporaryTables() throws Exception {
  FakeDatasetService datasetService = new FakeDatasetService();
  String projectId = "project";
  String datasetId = "dataset";
  datasetService.createDataset(projectId, datasetId, "", "", null);
  List<TableReference> tableRefs = Lists.newArrayList(
      BigQueryHelpers.parseTableSpec(String.format("%s:%s.%s", projectId, datasetId, "table1")),
      BigQueryHelpers.parseTableSpec(String.format("%s:%s.%s", projectId, datasetId, "table2")),
      BigQueryHelpers.parseTableSpec(String.format("%s:%s.%s", projectId, datasetId, "table3")));
  for (TableReference tableRef : tableRefs) {
    datasetService.createTable(new Table().setTableReference(tableRef));
  }

  // Add one more table to delete that does not actually exist.
  tableRefs.add(
      BigQueryHelpers.parseTableSpec(String.format("%s:%s.%s", projectId, datasetId, "table4")));

  WriteRename.removeTemporaryTables(datasetService, tableRefs);

  for (TableReference ref : tableRefs) {
    loggedWriteRename.verifyDebug("Deleting table " + toJsonString(ref));
    checkState(datasetService.getTable(ref) == null,
        "Table " + ref + " was not deleted!");
  }
}
 

import com.google.api.services.bigquery.model.TableReference; //导入依赖的package包/类
private Job makeLoadJob(JobReference jobRef, String sourceUri, String tableId) {
  TableReference tableReference = new TableReference()
      .setProjectId(jobRef.getProjectId())
      .setDatasetId(SNAPSHOTS_DATASET)
      .setTableId(tableId);
  return new Job()
      .setJobReference(jobRef)
      .setConfiguration(new JobConfiguration()
          .setLoad(new JobConfigurationLoad()
              .setWriteDisposition(WriteDisposition.WRITE_EMPTY.toString())
              .setSourceFormat(SourceFormat.DATASTORE_BACKUP.toString())
              .setSourceUris(ImmutableList.of(sourceUri))
              .setDestinationTable(tableReference)));
}
 

import com.google.api.services.bigquery.model.TableReference; //导入依赖的package包/类
@Override
public void validate(PipelineOptions pipelineOptions) {
  BigQueryOptions options = pipelineOptions.as(BigQueryOptions.class);

  // The user specified a table.
  if (getJsonTableRef() != null && getJsonTableRef().isAccessible() && getValidate()) {
    TableReference table = getTableWithDefaultProject(options).get();
    DatasetService datasetService = getBigQueryServices().getDatasetService(options);
    // Check for destination table presence and emptiness for early failure notification.
    // Note that a presence check can fail when the table or dataset is created by an earlier
    // stage of the pipeline. For these cases the #withoutValidation method can be used to
    // disable the check.
    BigQueryHelpers.verifyDatasetPresence(datasetService, table);
    if (getCreateDisposition() == BigQueryIO.Write.CreateDisposition.CREATE_NEVER) {
      BigQueryHelpers.verifyTablePresence(datasetService, table);
    }
    if (getWriteDisposition() == BigQueryIO.Write.WriteDisposition.WRITE_EMPTY) {
      BigQueryHelpers.verifyTableNotExistOrEmpty(datasetService, table);
    }
  }
}
 

import com.google.api.services.bigquery.model.TableReference; //导入依赖的package包/类
/**
 * Returns the table to write, or {@code null} if writing with {@code tableFunction}.
 *
 * <p>If the table's project is not specified, use the executing project.
 */
@Nullable
ValueProvider<TableReference> getTableWithDefaultProject(BigQueryOptions bqOptions) {
  ValueProvider<TableReference> table = getTable();
  if (table == null) {
    return table;
  }

  if (!table.isAccessible()) {
    LOG.info("Using a dynamic value for table input. This must contain a project"
            + " in the table reference: {}", table);
    return table;
  }
  if (Strings.isNullOrEmpty(table.get().getProjectId())) {
    // If user does not specify a project we assume the table to be located in
    // the default project.
    TableReference tableRef = table.get();
    tableRef.setProjectId(bqOptions.getProject());
    return NestedValueProvider.of(StaticValueProvider.of(
        BigQueryHelpers.toJsonString(tableRef)), new JsonTableRefToTableRef());
  }
  return table;
}
 

import com.google.api.services.bigquery.model.TableReference; //导入依赖的package包/类
/** Writes the accumulated rows into BigQuery with streaming API. */
@FinishBundle
public void finishBundle(FinishBundleContext context) throws Exception {
  List<ValueInSingleWindow<TableRow>> failedInserts = Lists.newArrayList();
  BigQueryOptions options = context.getPipelineOptions().as(BigQueryOptions.class);
  for (Map.Entry<String, List<ValueInSingleWindow<TableRow>>> entry : tableRows.entrySet()) {
    TableReference tableReference = BigQueryHelpers.parseTableSpec(entry.getKey());
    flushRows(
        tableReference,
        entry.getValue(),
        uniqueIdsForTableRows.get(entry.getKey()),
        options,
        failedInserts);
  }
  tableRows.clear();
  uniqueIdsForTableRows.clear();

  for (ValueInSingleWindow<TableRow> row : failedInserts) {
    context.output(failedOutputTag, row.getValue(), row.getTimestamp(), row.getWindow());
  }
}
 

import com.google.api.services.bigquery.model.TableReference; //导入依赖的package包/类
/**
 * Updates the specified Bigquery table to reflect the metadata from the input.
 *
 * <p>Returns the input DestinationTable. If the specified table does not already exist, it will
 * be inserted into the dataset.
 *
 * <p>Clients can call this function directly to update a table on demand, or can pass it to
 * Futures.transform() to update a table produced as the asynchronous result of a load or query
 * job (e.g. to add a description to it).
 */
private DestinationTable updateTable(final DestinationTable destinationTable) {
  Table table = destinationTable.getTable();
  TableReference ref = table.getTableReference();
  try {
    if (checkTableExists(ref.getDatasetId(), ref.getTableId())) {
      // Make sure to use patch() rather than update(). The former changes only those properties
      // which are specified, while the latter would change everything, blanking out unspecified
      // properties.
      bigquery
          .tables()
          .patch(ref.getProjectId(), ref.getDatasetId(), ref.getTableId(), table)
          .execute();
    } else {
      bigquery.tables().insert(ref.getProjectId(), ref.getDatasetId(), table).execute();
    }
    return destinationTable;
  } catch (IOException e) {
    throw BigqueryJobFailureException.create(e);
  }
}
 

import com.google.api.services.bigquery.model.TableReference; //导入依赖的package包/类
@Override
public synchronized long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
  if (tableSizeBytes.get() == null) {
    TableReference table = setDefaultProjectIfAbsent(options.as(BigQueryOptions.class),
        BigQueryIO.JSON_FACTORY.fromString(jsonTable.get(), TableReference.class));

    Table tableRef = bqServices.getDatasetService(options.as(BigQueryOptions.class))
            .getTable(table);
    Long numBytes = tableRef.getNumBytes();
    if (tableRef.getStreamingBuffer() != null) {
      numBytes += tableRef.getStreamingBuffer().getEstimatedBytes().longValue();
    }

    tableSizeBytes.compareAndSet(null, numBytes);
  }
  return tableSizeBytes.get();
}