Java TableSchema类代码示例

import com.google.api.services.bigquery.model.TableSchema; //导入依赖的package包/类
public static void main(String[] args) {
  Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);

  Pipeline p = Pipeline.create(options);

  // Build the table schema for the output table.
  List<TableFieldSchema> fields = new ArrayList<>();
  fields.add(new TableFieldSchema().setName("month").setType("INTEGER"));
  fields.add(new TableFieldSchema().setName("tornado_count").setType("INTEGER"));
  TableSchema schema = new TableSchema().setFields(fields);

  p.apply(BigQueryIO.readTableRows().from(options.getInput()))
   .apply(new CountTornadoes())
   .apply(BigQueryIO.writeTableRows()
       .to(options.getOutput())
       .withSchema(schema)
       .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED)
       .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE));

  p.run().waitUntilFinish();
}
 

import com.google.api.services.bigquery.model.TableSchema; //导入依赖的package包/类
/**
 * Setup step {A}
 * Helper method that defines the BigQuery schema used for the output.
 */
private static TableSchema getWebResourceSchema() {
	List<TableFieldSchema> fields = new ArrayList<>();
	fields.add(new TableFieldSchema().setName("WebResourceHash").setType("STRING").setMode("REQUIRED"));
	fields.add(new TableFieldSchema().setName("Url").setType("STRING"));
	fields.add(new TableFieldSchema().setName("PublicationTime").setType("TIMESTAMP").setMode("REQUIRED"));
	fields.add(new TableFieldSchema().setName("PublicationDateId").setType("INTEGER").setMode("REQUIRED"));
	fields.add(new TableFieldSchema().setName("ProcessingTime").setType("TIMESTAMP").setMode("REQUIRED"));
	fields.add(new TableFieldSchema().setName("ProcessingDateId").setType("INTEGER").setMode("REQUIRED"));
	fields.add(new TableFieldSchema().setName("DocumentHash").setType("STRING"));
	fields.add(new TableFieldSchema().setName("DocumentCollectionId").setType("STRING"));
	fields.add(new TableFieldSchema().setName("CollectionItemId").setType("STRING"));
	fields.add(new TableFieldSchema().setName("Title").setType("STRING"));
	fields.add(new TableFieldSchema().setName("Domain").setType("STRING"));
	fields.add(new TableFieldSchema().setName("Author").setType("STRING"));
	fields.add(new TableFieldSchema().setName("ParentWebResourceHash").setType("STRING"));
	fields.add(new TableFieldSchema().setName("MetaFields").setType("STRING").setMode("REPEATED"));

	TableSchema schema = new TableSchema().setFields(fields);
	return schema;
}
 

import com.google.api.services.bigquery.model.TableSchema; //导入依赖的package包/类
private TableSchema exampleTable() {
  TableSchema personTable = new TableSchema();
  personTable.setFields(Arrays.asList(
      new TableFieldSchema().setName("__key__").setType("STRING"),
      new TableFieldSchema().setName("firstName").setType("STRING"),
      new TableFieldSchema().setName("age").setType("INTEGER"),
      new TableFieldSchema().setName("pic").setType("BYTES"),
      new TableFieldSchema().setName("weight").setType("FLOAT"),
      new TableFieldSchema().setName("isGoogler").setType("BOOLEAN"),
      new TableFieldSchema().setName("birthTimestamp").setType("TIMESTAMP"),
      new TableFieldSchema().setName("birthDate").setType("DATE"),
      new TableFieldSchema().setName("birthTime").setType("TIME"),
      new TableFieldSchema().setName("birthDateTime").setType("DATETIME"),
      new TableFieldSchema().setName("favFruits").setType("STRING").setMode("REPEATED"),
      new TableFieldSchema().setName("body").setType("RECORD").setFields(Arrays.asList(
          new TableFieldSchema().setName("height").setType("FLOAT"),
          new TableFieldSchema().setName("ethnicity").setType("STRING")
      )),
      new TableFieldSchema().setName("vacations").setType("RECORD").setMode("REPEATED")
          .setFields(Arrays.asList(
              new TableFieldSchema().setName("place").setType("STRING"),
              new TableFieldSchema().setName("time").setType("TIMESTAMP")
          ))
  ));
  return personTable;
}
 

import com.google.api.services.bigquery.model.TableSchema; //导入依赖的package包/类
@Override
public TableSchema getSchema(String tableDefinition) {
	final String tableFullName = tableDefinition.substring(tableDefinition.indexOf('.') + 1,
			tableDefinition.length());
	_log.info("tableDefinition - table full name: " + tableFullName);

	List<MsSqlColumn> tableColumns = null;
	for (MsSqlTable table : columnsMap.keySet()) {
		if (table.getFullName().equals(tableFullName)) {
			tableColumns = columnsMap.get(table);
			break;
		}
	}

	TableSchema tableSchema = null;
	try {
		tableSchema = BigQueryUtils.getBigQuerySchema(tableColumns);
	} catch (SQLException e) {
		_log.error("Unable to translate column list to TableSchema, tableDefinition: " + tableDefinition, e);
	}
	_log.info("tableDefinition - is table schema null? " + (tableSchema == null));
	return tableSchema;
}
 

import com.google.api.services.bigquery.model.TableSchema; //导入依赖的package包/类
private void setupBigQueryTable(String projectId, String datasetId, String tableId,
    TableSchema schema) throws IOException {
  if (bigQueryClient == null) {
    bigQueryClient = Transport.newBigQueryClient(options.as(BigQueryOptions.class)).build();
  }

  Datasets datasetService = bigQueryClient.datasets();
  if (executeNullIfNotFound(datasetService.get(projectId, datasetId)) == null) {
    Dataset newDataset = new Dataset().setDatasetReference(
        new DatasetReference().setProjectId(projectId).setDatasetId(datasetId));
    datasetService.insert(projectId, newDataset).execute();
  }

  Tables tableService = bigQueryClient.tables();
  Table table = executeNullIfNotFound(tableService.get(projectId, datasetId, tableId));
  if (table == null) {
    Table newTable = new Table().setSchema(schema).setTableReference(
        new TableReference().setProjectId(projectId).setDatasetId(datasetId).setTableId(tableId));
    tableService.insert(projectId, datasetId, newTable).execute();
  } else if (!table.getSchema().equals(schema)) {
    throw new RuntimeException(
        "Table exists and schemas do not match, expecting: " + schema.toPrettyString()
        + ", actual: " + table.getSchema().toPrettyString());
  }
}
 

import com.google.api.services.bigquery.model.TableSchema; //导入依赖的package包/类
/** Defines the BigQuery schema used for the output. */
static TableSchema getSchema() {
  List<TableFieldSchema> fields = new ArrayList<>();
  fields.add(new TableFieldSchema().setName("station_id").setType("STRING"));
  fields.add(new TableFieldSchema().setName("direction").setType("STRING"));
  fields.add(new TableFieldSchema().setName("freeway").setType("STRING"));
  fields.add(new TableFieldSchema().setName("lane_max_flow").setType("INTEGER"));
  fields.add(new TableFieldSchema().setName("lane").setType("STRING"));
  fields.add(new TableFieldSchema().setName("avg_occ").setType("FLOAT"));
  fields.add(new TableFieldSchema().setName("avg_speed").setType("FLOAT"));
  fields.add(new TableFieldSchema().setName("total_flow").setType("INTEGER"));
  fields.add(new TableFieldSchema().setName("window_timestamp").setType("TIMESTAMP"));
  fields.add(new TableFieldSchema().setName("recorded_timestamp").setType("STRING"));
  TableSchema schema = new TableSchema().setFields(fields);
  return schema;
}
 

import com.google.api.services.bigquery.model.TableSchema; //导入依赖的package包/类
/** Defines the BigQuery schema used for the output. */
private static TableSchema getSchema() {
  List<TableFieldSchema> fields = new ArrayList<>();
  fields.add(new TableFieldSchema().setName("trigger_type").setType("STRING"));
  fields.add(new TableFieldSchema().setName("freeway").setType("STRING"));
  fields.add(new TableFieldSchema().setName("total_flow").setType("INTEGER"));
  fields.add(new TableFieldSchema().setName("number_of_records").setType("INTEGER"));
  fields.add(new TableFieldSchema().setName("window").setType("STRING"));
  fields.add(new TableFieldSchema().setName("isFirst").setType("BOOLEAN"));
  fields.add(new TableFieldSchema().setName("isLast").setType("BOOLEAN"));
  fields.add(new TableFieldSchema().setName("timing").setType("STRING"));
  fields.add(new TableFieldSchema().setName("event_time").setType("TIMESTAMP"));
  fields.add(new TableFieldSchema().setName("processing_time").setType("TIMESTAMP"));
  TableSchema schema = new TableSchema().setFields(fields);
  return schema;
}
 

import com.google.api.services.bigquery.model.TableSchema; //导入依赖的package包/类
public static void main(String[] args)
    throws Exception {

  Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
  Pipeline p = Pipeline.create(options);

  TableSchema schema = buildWeatherSchemaProjection();

  p.apply(BigQueryIO.readTableRows().from(options.getInput()))
   .apply(ParDo.of(new ProjectionFn()))
   .apply(new BelowGlobalMean(options.getMonthFilter()))
   .apply(BigQueryIO.writeTableRows()
      .to(options.getOutput())
      .withSchema(schema)
      .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED)
      .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE));

  p.run().waitUntilFinish();
}
 

import com.google.api.services.bigquery.model.TableSchema; //导入依赖的package包/类
public static void main(String[] args)
    throws Exception {

  Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
  Pipeline p = Pipeline.create(options);

  // Build the table schema for the output table.
  List<TableFieldSchema> fields = new ArrayList<>();
  fields.add(new TableFieldSchema().setName("word").setType("STRING"));
  fields.add(new TableFieldSchema().setName("all_plays").setType("STRING"));
  TableSchema schema = new TableSchema().setFields(fields);

  p.apply(BigQueryIO.readTableRows().from(options.getInput()))
   .apply(new PlaysForWord())
   .apply(BigQueryIO.writeTableRows()
      .to(options.getOutput())
      .withSchema(schema)
      .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED)
      .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE));

  p.run().waitUntilFinish();
}
 

import com.google.api.services.bigquery.model.TableSchema; //导入依赖的package包/类
public static void main(String[] args)
    throws Exception {

  Options options = PipelineOptionsFactory.fromArgs(args).withValidation().as(Options.class);
  Pipeline p = Pipeline.create(options);

  // Build the table schema for the output table.
  List<TableFieldSchema> fields = new ArrayList<>();
  fields.add(new TableFieldSchema().setName("month").setType("INTEGER"));
  fields.add(new TableFieldSchema().setName("max_mean_temp").setType("FLOAT"));
  TableSchema schema = new TableSchema().setFields(fields);

  p.apply(BigQueryIO.readTableRows().from(options.getInput()))
   .apply(new MaxMeanTemp())
   .apply(BigQueryIO.writeTableRows()
      .to(options.getOutput())
      .withSchema(schema)
      .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED)
      .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE));

  p.run().waitUntilFinish();
}
 

import com.google.api.services.bigquery.model.TableSchema; //导入依赖的package包/类
private void setupBigQueryTable(String projectId, String datasetId, String tableId,
    TableSchema schema) throws IOException {
  if (bigQueryClient == null) {
    bigQueryClient = newBigQueryClient(options.as(BigQueryOptions.class)).build();
  }

  Datasets datasetService = bigQueryClient.datasets();
  if (executeNullIfNotFound(datasetService.get(projectId, datasetId)) == null) {
    Dataset newDataset = new Dataset().setDatasetReference(
        new DatasetReference().setProjectId(projectId).setDatasetId(datasetId));
    datasetService.insert(projectId, newDataset).execute();
  }

  Tables tableService = bigQueryClient.tables();
  Table table = executeNullIfNotFound(tableService.get(projectId, datasetId, tableId));
  if (table == null) {
    Table newTable = new Table().setSchema(schema).setTableReference(
        new TableReference().setProjectId(projectId).setDatasetId(datasetId).setTableId(tableId));
    tableService.insert(projectId, datasetId, newTable).execute();
  } else if (!table.getSchema().equals(schema)) {
    throw new RuntimeException(
        "Table exists and schemas do not match, expecting: " + schema.toPrettyString()
        + ", actual: " + table.getSchema().toPrettyString());
  }
}
 

import com.google.api.services.bigquery.model.TableSchema; //导入依赖的package包/类
/**
 * Send {@code formattedResults} to BigQuery.
 */
private void sinkResultsToBigQuery(
    PCollection<String> formattedResults, long now,
    String version) {
  String tableSpec = tableSpec(now, version);
  TableSchema tableSchema =
      new TableSchema().setFields(ImmutableList.of(
          new TableFieldSchema().setName("result").setType("STRING"),
          new TableFieldSchema().setName("records").setMode("REPEATED").setType("RECORD")
                                .setFields(ImmutableList.of(
                                    new TableFieldSchema().setName("index").setType("INTEGER"),
                                    new TableFieldSchema().setName("value").setType("STRING")))));
  NexmarkUtils.console("Writing results to BigQuery table %s", tableSpec);
  BigQueryIO.Write io =
      BigQueryIO.write().to(tableSpec)
                      .withSchema(tableSchema)
                      .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED)
                      .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND);
  formattedResults
      .apply(queryName + ".StringToTableRow", ParDo.of(new StringToTableRow()))
      .apply(queryName + ".WriteBigQueryResults", io);
}
 

import com.google.api.services.bigquery.model.TableSchema; //导入依赖的package包/类
protected ExtractResult extractFiles(PipelineOptions options) throws Exception {
  BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
  TableReference tableToExtract = getTableToExtract(bqOptions);
  Table table = bqServices.getDatasetService(bqOptions).getTable(tableToExtract);
  if (table == null) {
    throw new IOException(String.format(
            "Cannot start an export job since table %s does not exist",
            BigQueryHelpers.toTableSpec(tableToExtract)));
  }

  TableSchema schema = table.getSchema();
  JobService jobService = bqServices.getJobService(bqOptions);
  String extractJobId = getExtractJobId(createJobIdToken(options.getJobName(), stepUuid));
  final String extractDestinationDir =
      resolveTempLocation(bqOptions.getTempLocation(), "BigQueryExtractTemp", stepUuid);
  List<ResourceId> tempFiles =
      executeExtract(
          extractJobId,
          tableToExtract,
          jobService,
          bqOptions.getProject(),
          extractDestinationDir);
  return new ExtractResult(schema, tempFiles);
}
 

import com.google.api.services.bigquery.model.TableSchema; //导入依赖的package包/类
List<BoundedSource<T>> createSources(List<ResourceId> files, TableSchema schema)
    throws IOException, InterruptedException {

  final String jsonSchema = BigQueryIO.JSON_FACTORY.toString(schema);
  SerializableFunction<GenericRecord, T> fnWrapper =
      new SerializableFunction<GenericRecord, T>() {
        private Supplier<TableSchema> schema = Suppliers.memoize(
            Suppliers.compose(new TableSchemaFunction(), Suppliers.ofInstance(jsonSchema)));

        @Override
        public T apply(GenericRecord input) {
          return parseFn.apply(new SchemaAndRecord(input, schema.get()));
        }
      };
  List<BoundedSource<T>> avroSources = Lists.newArrayList();
  for (ResourceId file : files) {
    avroSources.add(
        AvroSource.from(file.toString()).withParseFn(fnWrapper, getOutputCoder()));
  }
  return ImmutableList.copyOf(avroSources);
}
 

import com.google.api.services.bigquery.model.TableSchema; //导入依赖的package包/类
/**
 * Tests that table creation succeeds when the table already exists.
 */
@Test
public void testCreateTableSucceedsAlreadyExists() throws IOException {
  TableReference ref =
      new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
  TableSchema schema = new TableSchema().setFields(ImmutableList.of(
      new TableFieldSchema().setName("column1").setType("String"),
      new TableFieldSchema().setName("column2").setType("Integer")));
  Table testTable = new Table().setTableReference(ref).setSchema(schema);

  when(response.getStatusCode()).thenReturn(409); // 409 means already exists

  BigQueryServicesImpl.DatasetServiceImpl services =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
  Table ret =
      services.tryCreateTable(
          testTable,
          new RetryBoundedBackOff(0, BackOff.ZERO_BACKOFF),
          Sleeper.DEFAULT);

  assertNull(ret);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}