Java BigQueryIO.Write方法代码示例

import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; //导入方法依赖的package包/类
/**
 * Send {@code formattedResults} to BigQuery.
 */
private void sinkResultsToBigQuery(
    PCollection<String> formattedResults, long now,
    String version) {
  String tableSpec = tableSpec(now, version);
  TableSchema tableSchema =
      new TableSchema().setFields(ImmutableList.of(
          new TableFieldSchema().setName("result").setType("STRING"),
          new TableFieldSchema().setName("records").setMode("REPEATED").setType("RECORD")
                                .setFields(ImmutableList.of(
                                    new TableFieldSchema().setName("index").setType("INTEGER"),
                                    new TableFieldSchema().setName("value").setType("STRING")))));
  NexmarkUtils.console("Writing results to BigQuery table %s", tableSpec);
  BigQueryIO.Write io =
      BigQueryIO.write().to(tableSpec)
                      .withSchema(tableSchema)
                      .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED)
                      .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND);
  formattedResults
      .apply(queryName + ".StringToTableRow", ParDo.of(new StringToTableRow()))
      .apply(queryName + ".WriteBigQueryResults", io);
}
 

import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; //导入方法依赖的package包/类
private BigQueryIO.Write setWriteOperation(BigQueryIO.Write bigQueryIOPTransform) {
    if (properties.tableOperation.getValue() == BigQueryOutputProperties.TableOperation.NONE
            || properties.tableOperation.getValue() == BigQueryOutputProperties.TableOperation.CREATE_IF_NOT_EXISTS) {
        switch (properties.writeOperation.getValue()) {
        case APPEND:
            bigQueryIOPTransform = bigQueryIOPTransform.withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND);
            break;
        case WRITE_TO_EMPTY:
            bigQueryIOPTransform = bigQueryIOPTransform.withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_EMPTY);
            break;
        default:
            throw new RuntimeException("To be implemented: " + properties.writeOperation.getValue());
        }
    } else {
        if (properties.writeOperation.getValue() != null) {
            LOG.info("Write operation " + properties.writeOperation.getValue() + " be ignored when Table operation is "
                    + properties.tableOperation.getValue());
        }
    }
    return bigQueryIOPTransform;
}
 

import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; //导入方法依赖的package包/类
@Override
public PDone expand(PCollection<IndexedRecord> in) {
    TableReference table = new TableReference();
    table.setProjectId(datastore.projectName.getValue());
    table.setDatasetId(dataset.bqDataset.getValue());
    table.setTableId(dataset.tableName.getValue());

    BigQueryIO.Write bigQueryIOPTransform = BigQueryIO.writeTableRows().to(table);

    bigQueryIOPTransform = setTableOperation(bigQueryIOPTransform);
    bigQueryIOPTransform = setWriteOperation(bigQueryIOPTransform);

    in.apply(ParDo.of(new IndexedRecordToTableRowFn())).apply(bigQueryIOPTransform);
    return PDone.in(in.getPipeline());
}
 

import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO; //导入方法依赖的package包/类
private BigQueryIO.Write setTableOperation(BigQueryIO.Write bigQueryIOPTransform) {
    TableSchema bqSchema = null;
    if (properties.tableOperation.getValue() == BigQueryOutputProperties.TableOperation.CREATE_IF_NOT_EXISTS
            || properties.tableOperation.getValue() == BigQueryOutputProperties.TableOperation.DROP_IF_EXISTS_AND_CREATE) {
        Schema designSchema = properties.getDatasetProperties().main.schema.getValue();
        if (designSchema != null && !AvroUtils.isSchemaEmpty(designSchema) && !AvroUtils.isIncludeAllFields(designSchema)) {
            bqSchema = BigQueryAvroRegistry.get().guessBigQuerySchema(designSchema);
        }
        if (bqSchema == null) {
            throw new RuntimeException("Need to specify schema to create table");
        }
    }

    switch (properties.tableOperation.getValue()) {
    case NONE:
        bigQueryIOPTransform = bigQueryIOPTransform.withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_NEVER);
        break;
    case CREATE_IF_NOT_EXISTS:
        bigQueryIOPTransform = bigQueryIOPTransform.withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED)
                .withSchema(bqSchema);
        break;
    case DROP_IF_EXISTS_AND_CREATE:
        bigQueryIOPTransform = bigQueryIOPTransform.withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE)
                .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED).withSchema(bqSchema);
        break;
    case TRUNCATE:
        bigQueryIOPTransform = bigQueryIOPTransform.withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE)
                .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_NEVER);
        break;
    default:
        throw new RuntimeException("To be implemented: " + properties.tableOperation.getValue());
    }
    return bigQueryIOPTransform;
}