Java Table类代码示例

import com.google.api.services.bigquery.model.Table; //导入依赖的package包/类
private void setupBigQueryTable(String projectId, String datasetId, String tableId,
    TableSchema schema) throws IOException {
  if (bigQueryClient == null) {
    bigQueryClient = Transport.newBigQueryClient(options.as(BigQueryOptions.class)).build();
  }

  Datasets datasetService = bigQueryClient.datasets();
  if (executeNullIfNotFound(datasetService.get(projectId, datasetId)) == null) {
    Dataset newDataset = new Dataset().setDatasetReference(
        new DatasetReference().setProjectId(projectId).setDatasetId(datasetId));
    datasetService.insert(projectId, newDataset).execute();
  }

  Tables tableService = bigQueryClient.tables();
  Table table = executeNullIfNotFound(tableService.get(projectId, datasetId, tableId));
  if (table == null) {
    Table newTable = new Table().setSchema(schema).setTableReference(
        new TableReference().setProjectId(projectId).setDatasetId(datasetId).setTableId(tableId));
    tableService.insert(projectId, datasetId, newTable).execute();
  } else if (!table.getSchema().equals(schema)) {
    throw new RuntimeException(
        "Table exists and schemas do not match, expecting: " + schema.toPrettyString()
        + ", actual: " + table.getSchema().toPrettyString());
  }
}
 

import com.google.api.services.bigquery.model.Table; //导入依赖的package包/类
Optional<Table> getTable(String projectId, String datasetId, String tableId,  boolean rowsExist)
        throws IOException
{
    try {
        Table ret = client.tables().get(projectId, datasetId, tableId).execute();
        if(rowsExist && ret.getNumRows().compareTo(BigInteger.ZERO) <= 0) {
            return Optional.absent();
        }
        return Optional.of(ret);
    }
    catch (GoogleJsonResponseException e) {
        if (e.getStatusCode() == HttpStatusCodes.STATUS_CODE_NOT_FOUND) {
            return Optional.absent();
        }
        throw e;
    }
}
 

import com.google.api.services.bigquery.model.Table; //导入依赖的package包/类
private void setupBigQueryTable(String projectId, String datasetId, String tableId,
    TableSchema schema) throws IOException {
  if (bigQueryClient == null) {
    bigQueryClient = newBigQueryClient(options.as(BigQueryOptions.class)).build();
  }

  Datasets datasetService = bigQueryClient.datasets();
  if (executeNullIfNotFound(datasetService.get(projectId, datasetId)) == null) {
    Dataset newDataset = new Dataset().setDatasetReference(
        new DatasetReference().setProjectId(projectId).setDatasetId(datasetId));
    datasetService.insert(projectId, newDataset).execute();
  }

  Tables tableService = bigQueryClient.tables();
  Table table = executeNullIfNotFound(tableService.get(projectId, datasetId, tableId));
  if (table == null) {
    Table newTable = new Table().setSchema(schema).setTableReference(
        new TableReference().setProjectId(projectId).setDatasetId(datasetId).setTableId(tableId));
    tableService.insert(projectId, datasetId, newTable).execute();
  } else if (!table.getSchema().equals(schema)) {
    throw new RuntimeException(
        "Table exists and schemas do not match, expecting: " + schema.toPrettyString()
        + ", actual: " + table.getSchema().toPrettyString());
  }
}
 

import com.google.api.services.bigquery.model.Table; //导入依赖的package包/类
@Override
public synchronized long getEstimatedSizeBytes(PipelineOptions options) throws Exception {
  if (tableSizeBytes.get() == null) {
    TableReference table = setDefaultProjectIfAbsent(options.as(BigQueryOptions.class),
        BigQueryIO.JSON_FACTORY.fromString(jsonTable.get(), TableReference.class));

    Table tableRef = bqServices.getDatasetService(options.as(BigQueryOptions.class))
            .getTable(table);
    Long numBytes = tableRef.getNumBytes();
    if (tableRef.getStreamingBuffer() != null) {
      numBytes += tableRef.getStreamingBuffer().getEstimatedBytes().longValue();
    }

    tableSizeBytes.compareAndSet(null, numBytes);
  }
  return tableSizeBytes.get();
}
 

import com.google.api.services.bigquery.model.Table; //导入依赖的package包/类
@VisibleForTesting
@Nullable
Table getTable(TableReference ref, BackOff backoff, Sleeper sleeper)
    throws IOException, InterruptedException {
  try {
    return executeWithRetries(
        client.tables().get(ref.getProjectId(), ref.getDatasetId(), ref.getTableId()),
        String.format(
            "Unable to get table: %s, aborting after %d retries.",
            ref.getTableId(), MAX_RPC_RETRIES),
        sleeper,
        backoff,
        DONT_RETRY_NOT_FOUND);
  } catch (IOException e) {
    if (errorExtractor.itemNotFound(e)) {
      return null;
    }
    throw e;
  }
}
 

import com.google.api.services.bigquery.model.Table; //导入依赖的package包/类
@Override
public Table patchTableDescription(TableReference tableReference,
                                   @Nullable String tableDescription)
    throws IOException, InterruptedException {
  Table table = new Table();
  table.setDescription(tableDescription);

  return executeWithRetries(
      client.tables().patch(
          tableReference.getProjectId(),
          tableReference.getDatasetId(),
          tableReference.getTableId(),
          table),
      String.format(
          "Unable to patch table description: %s, aborting after %d retries.",
          tableReference, MAX_RPC_RETRIES),
      Sleeper.DEFAULT,
      createDefaultBackoff(),
      ALWAYS_RETRY);
}
 

import com.google.api.services.bigquery.model.Table; //导入依赖的package包/类
protected ExtractResult extractFiles(PipelineOptions options) throws Exception {
  BigQueryOptions bqOptions = options.as(BigQueryOptions.class);
  TableReference tableToExtract = getTableToExtract(bqOptions);
  Table table = bqServices.getDatasetService(bqOptions).getTable(tableToExtract);
  if (table == null) {
    throw new IOException(String.format(
            "Cannot start an export job since table %s does not exist",
            BigQueryHelpers.toTableSpec(tableToExtract)));
  }

  TableSchema schema = table.getSchema();
  JobService jobService = bqServices.getJobService(bqOptions);
  String extractJobId = getExtractJobId(createJobIdToken(options.getJobName(), stepUuid));
  final String extractDestinationDir =
      resolveTempLocation(bqOptions.getTempLocation(), "BigQueryExtractTemp", stepUuid);
  List<ResourceId> tempFiles =
      executeExtract(
          extractJobId,
          tableToExtract,
          jobService,
          bqOptions.getProject(),
          extractDestinationDir);
  return new ExtractResult(schema, tempFiles);
}
 

import com.google.api.services.bigquery.model.Table; //导入依赖的package包/类
@Test
public void testGetTableSucceeds() throws Exception {
  TableReference tableRef = new TableReference()
      .setProjectId("projectId")
      .setDatasetId("datasetId")
      .setTableId("tableId");

  Table testTable = new Table();
  testTable.setTableReference(tableRef);

  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(403).thenReturn(200);
  when(response.getContent())
      .thenReturn(toStream(errorWithReasonAndStatus("rateLimitExceeded", 403)))
      .thenReturn(toStream(testTable));

  BigQueryServicesImpl.DatasetServiceImpl datasetService =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());

  Table table = datasetService.getTable(tableRef, BackOff.ZERO_BACKOFF, Sleeper.DEFAULT);

  assertEquals(testTable, table);
  verify(response, times(2)).getStatusCode();
  verify(response, times(2)).getContent();
  verify(response, times(2)).getContentType();
}
 

import com.google.api.services.bigquery.model.Table; //导入依赖的package包/类
@Test
public void testGetTableNotFound() throws IOException, InterruptedException {
  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(404);

  BigQueryServicesImpl.DatasetServiceImpl datasetService =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());

  TableReference tableRef = new TableReference()
      .setProjectId("projectId")
      .setDatasetId("datasetId")
      .setTableId("tableId");
  Table table = datasetService.getTable(tableRef, BackOff.ZERO_BACKOFF, Sleeper.DEFAULT);

  assertNull(table);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 

import com.google.api.services.bigquery.model.Table; //导入依赖的package包/类
@Test
public void testExecuteWithRetries() throws IOException, InterruptedException {
  Table testTable = new Table();

  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(200);
  when(response.getContent()).thenReturn(toStream(testTable));

  Table table = BigQueryServicesImpl.executeWithRetries(
      bigquery.tables().get("projectId", "datasetId", "tableId"),
      "Failed to get table.",
      Sleeper.DEFAULT,
      BackOff.STOP_BACKOFF,
      BigQueryServicesImpl.ALWAYS_RETRY);

  assertEquals(testTable, table);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 

import com.google.api.services.bigquery.model.Table; //导入依赖的package包/类
@Test
public void testCreateTableSucceeds() throws IOException {
  TableReference ref =
      new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
  Table testTable = new Table().setTableReference(ref);
  when(response.getContentType()).thenReturn(Json.MEDIA_TYPE);
  when(response.getStatusCode()).thenReturn(200);
  when(response.getContent()).thenReturn(toStream(testTable));

  BigQueryServicesImpl.DatasetServiceImpl services =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
  Table ret =
      services.tryCreateTable(
          testTable,
          new RetryBoundedBackOff(0, BackOff.ZERO_BACKOFF),
          Sleeper.DEFAULT);
  assertEquals(testTable, ret);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 

import com.google.api.services.bigquery.model.Table; //导入依赖的package包/类
/**
 * Tests that table creation succeeds when the table already exists.
 */
@Test
public void testCreateTableSucceedsAlreadyExists() throws IOException {
  TableReference ref =
      new TableReference().setProjectId("project").setDatasetId("dataset").setTableId("table");
  TableSchema schema = new TableSchema().setFields(ImmutableList.of(
      new TableFieldSchema().setName("column1").setType("String"),
      new TableFieldSchema().setName("column2").setType("Integer")));
  Table testTable = new Table().setTableReference(ref).setSchema(schema);

  when(response.getStatusCode()).thenReturn(409); // 409 means already exists

  BigQueryServicesImpl.DatasetServiceImpl services =
      new BigQueryServicesImpl.DatasetServiceImpl(bigquery, PipelineOptionsFactory.create());
  Table ret =
      services.tryCreateTable(
          testTable,
          new RetryBoundedBackOff(0, BackOff.ZERO_BACKOFF),
          Sleeper.DEFAULT);

  assertNull(ret);
  verify(response, times(1)).getStatusCode();
  verify(response, times(1)).getContent();
  verify(response, times(1)).getContentType();
}
 

import com.google.api.services.bigquery.model.Table; //导入依赖的package包/类
private boolean validateDispositions(Table table, CreateDisposition createDisposition,
                                     WriteDisposition writeDisposition)
    throws InterruptedException, IOException {
  if (table == null) {
    if (createDisposition == CreateDisposition.CREATE_NEVER) {
      return false;
    }
  } else if (writeDisposition == WriteDisposition.WRITE_TRUNCATE) {
    datasetService.deleteTable(table.getTableReference());
  } else if (writeDisposition == WriteDisposition.WRITE_EMPTY) {
    List<TableRow> allRows = datasetService.getAllRows(table.getTableReference().getProjectId(),
        table.getTableReference().getDatasetId(), table.getTableReference().getTableId());
    if (!allRows.isEmpty()) {
      return false;
    }
  }
  return true;
}
 

import com.google.api.services.bigquery.model.Table; //导入依赖的package包/类
@Override
public Table getTable(TableReference tableRef)
    throws InterruptedException, IOException {
  synchronized (tables) {
    Map<String, TableContainer> dataset =
            tables.get(tableRef.getProjectId(), tableRef.getDatasetId());
    if (dataset == null) {
      throwNotFound(
          "Tried to get a dataset %s:%s, but no such dataset was set",
          tableRef.getProjectId(),
          tableRef.getDatasetId());
    }
    TableContainer tableContainer = dataset.get(tableRef.getTableId());
    return tableContainer == null ? null : tableContainer.getTable();
  }
}
 

import com.google.api.services.bigquery.model.Table; //导入依赖的package包/类
@Override
public void createTable(Table table) throws IOException {
  TableReference tableReference = table.getTableReference();
  validateWholeTableReference(tableReference);
  synchronized (tables) {
    Map<String, TableContainer> dataset =
        tables.get(tableReference.getProjectId(), tableReference.getDatasetId());
    if (dataset == null) {
      throwNotFound(
          "Tried to get a dataset %s:%s, but no such table was set",
          tableReference.getProjectId(),
          tableReference.getDatasetId());
    }
    TableContainer tableContainer = dataset.get(tableReference.getTableId());
    if (tableContainer == null) {
      tableContainer = new TableContainer(table);
      dataset.put(tableReference.getTableId(), tableContainer);
    }
  }
}