Java Bigquery类代码示例

import com.google.api.services.bigquery.Bigquery; //导入依赖的package包/类
public BigQueryExporter(BigQueryExporterConfiguration config) {
	this.config = config;
	this.checkedSchemas = new HashSet<String>();
	this.existingSchemaMap = new HashMap<String, com.google.api.services.bigquery.model.TableSchema>();

	HttpTransport transport = new NetHttpTransport();
	JsonFactory jsonFactory = new JacksonFactory();
	GoogleCredential credential;
	try {
		credential = GoogleCredential.getApplicationDefault(transport,
				jsonFactory);
	} catch (IOException e) {
		throw new RuntimeException(e);
	}
	if (credential.createScopedRequired()) {
		credential = credential.createScoped(BigqueryScopes.all());
	}
	this.bq = new Bigquery.Builder(transport, jsonFactory, credential)
			.setApplicationName(this.config.applicationName).build();
}
 

import com.google.api.services.bigquery.Bigquery; //导入依赖的package包/类
public static Iterator<GetQueryResultsResponse> run(String projectId, 
    String queryString, 
    long waitTime) throws IOException{
  Bigquery bigquery = BigqueryServiceFactory.getService();
  //Wait until query is done with 10 second timeout, at most 5 retries on error
  QueryResponse query = bigquery.jobs().query(
      projectId,
      new QueryRequest().setTimeoutMs(waitTime).setQuery(queryString)).execute();
  
  //Make a request to get the results of the query 
  //(timeout is zero since job should be complete)
  
  GetQueryResults getRequest = bigquery.jobs().getQueryResults(
      query.getJobReference().getProjectId(),
      query.getJobReference().getJobId());
  
  
  return getPages(getRequest);
}
 

import com.google.api.services.bigquery.Bigquery; //导入依赖的package包/类
public static void run( 
    String cloudStoragePath, 
    String projectId, 
    String datasetId, 
    String tableId,
    long interval) throws IOException, InterruptedException{

  Bigquery bigquery = BigqueryServiceFactory.getService();
  
  Job extractJob = extractJob(
      bigquery,
      cloudStoragePath,
      new TableReference()
      .setProjectId(projectId)
      .setDatasetId(datasetId)
      .setTableId(tableId));
  
  Bigquery.Jobs.Get get_job = bigquery.jobs().get(
      extractJob.getJobReference().getProjectId(), 
      extractJob.getJobReference().getJobId());
  
  pollJob(get_job, interval);
  
  System.out.println("Export is Done!");
  
}
 

import com.google.api.services.bigquery.Bigquery; //导入依赖的package包/类
@Before
public void setUp() {
  MockitoAnnotations.initMocks(this);

  // A mock transport that lets us mock the API responses.
  MockHttpTransport transport =
      new MockHttpTransport.Builder()
          .setLowLevelHttpRequest(
              new MockLowLevelHttpRequest() {
                @Override
                public LowLevelHttpResponse execute() throws IOException {
                  return response;
                }
              })
          .build();

  // A sample BigQuery API client that uses default JsonFactory and RetryHttpInitializer.
  bigquery =
      new Bigquery.Builder(
              transport, Transport.getJsonFactory(), new RetryHttpRequestInitializer())
          .build();
}
 

import com.google.api.services.bigquery.Bigquery; //导入依赖的package包/类
/**
 * Returns a new connection to Bigquery, first ensuring that the given dataset exists in the
 * project with the given id, creating it if required.
 */
public Bigquery create(String projectId, String datasetId) throws IOException {
  Bigquery bigquery = create(
      getClass().getSimpleName(),
      new UrlFetchTransport(),
      new JacksonFactory(),
      new AppIdentityCredential(BigqueryScopes.all()));

  // Note: it's safe for multiple threads to call this as the dataset will only be created once.
  if (!knownExistingDatasets.contains(datasetId)) {
    ensureDataset(bigquery, projectId, datasetId);
    knownExistingDatasets.add(datasetId);
  }

  return bigquery;
}
 

import com.google.api.services.bigquery.Bigquery; //导入依赖的package包/类
/**
 * Ensures the dataset exists by trying to create it. Note that it's not appreciably cheaper
 * to check for dataset existence than it is to try to create it and check for exceptions.
 */
// Note that these are not static so they can be mocked for testing.
private void ensureDataset(Bigquery bigquery, String projectId, String datasetId)
    throws IOException {
  try {
    bigquery.datasets()
        .insert(projectId,
            new Dataset().setDatasetReference(
                new DatasetReference()
                    .setProjectId(projectId)
                    .setDatasetId(datasetId)))
        .execute();
  } catch (IOException e) {
    // Swallow errors about a duplicate dataset, and throw any other ones.
    if (!BigqueryJobFailureException.create(e).getReason().equals("duplicate")) {
      throw e;
    }
  }
}
 

import com.google.api.services.bigquery.Bigquery; //导入依赖的package包/类
/** Ensures the table exists in Bigquery. */
private void ensureTable(Bigquery bigquery, TableReference table, List<TableFieldSchema> schema)
    throws IOException {
  try {
    bigquery.tables().insert(table.getProjectId(), table.getDatasetId(), new Table()
        .setSchema(new TableSchema().setFields(schema))
        .setTableReference(table))
        .execute();
    logger.infofmt("Created BigQuery table %s:%s.%s", table.getProjectId(), table.getDatasetId(),
        table.getTableId());
  } catch (IOException e) {
    // Swallow errors about a table that exists, and throw any other ones.
    if (!BigqueryJobFailureException.create(e).getReason().equals("duplicate")) {
      throw e;
    }
  }
}
 

import com.google.api.services.bigquery.Bigquery; //导入依赖的package包/类
@Test
public void test() throws IOException {
	String projectId = "";
	String datasetId = "ontologies";
	String tableId = "";
	String timestamp = Long.toString((new Date()).getTime());
	
	Bigquery bigquery = null;
	TableRow row = new TableRow();
	row.set("column_name", 7.7);
	TableDataInsertAllRequest.Rows rows = new TableDataInsertAllRequest.Rows();
	rows.setInsertId(timestamp);
	
	rows.setJson(row);
	List  rowList =
	    new ArrayList();
	rowList.add(rows);
	TableDataInsertAllRequest content = 
	    new TableDataInsertAllRequest().setRows(rowList);
	TableDataInsertAllResponse response =
	    bigquery.tabledata().insertAll(
	        projectId, datasetId, tableId, content).execute();
}
 

import com.google.api.services.bigquery.Bigquery; //导入依赖的package包/类
@Override
public void cancel(InterpreterContext context) {

  logger.info("Trying to Cancel current query statement.");

  if (service != null && jobId != null && projectId != null) {
    try {
      Bigquery.Jobs.Cancel request = service.jobs().cancel(projectId, jobId);
      JobCancelResponse response = request.execute();
      jobId = null;
      logger.info("Query Execution cancelled");
    } catch (IOException ex) {
      logger.error("Could not cancel the SQL execution");
    }
  } else {
    logger.info("Query Execution was already cancelled");
  }
}
 

import com.google.api.services.bigquery.Bigquery; //导入依赖的package包/类
public static Iterator<GetQueryResultsResponse> run(String projectId,
    String queryString,
    boolean batch, 
    long waitTime) 
    throws IOException, InterruptedException{
  
  Bigquery bigquery = BigqueryServiceFactory.getService();

  Job query = asyncQuery(bigquery, projectId, queryString, batch);
  Bigquery.Jobs.Get getRequest = bigquery.jobs().get(
      projectId, query.getJobReference().getJobId());
  
  //Poll every waitTime milliseconds, 
  //retrying at most retries times if there are errors
  pollJob(getRequest, waitTime);

  GetQueryResults resultsRequest = bigquery.jobs().getQueryResults(
      projectId, query.getJobReference().getJobId());
  
   return getPages(resultsRequest);
}
 

import com.google.api.services.bigquery.Bigquery; //导入依赖的package包/类
/**
 * Inserts an asynchronous query Job for a particular query
 *
 * @param bigquery  an authorized BigQuery client
 * @param projectId a String containing the project ID
 * @param querySql  the actual query string
 * @return a reference to the inserted query job
 * @throws IOException
 */
public static Job asyncQuery(Bigquery bigquery, 
    String projectId,
    String querySql,
    boolean batch) throws IOException {
  
  JobConfigurationQuery query_config = new JobConfigurationQuery()
        .setQuery(querySql);
  
  if(batch){
    query_config.setPriority("BATCH");
  }
  
  Job job = new Job().setConfiguration(
      new JobConfiguration().setQuery(query_config));
          
  return bigquery.jobs().insert(projectId, job).execute();
}
 

import com.google.api.services.bigquery.Bigquery; //导入依赖的package包/类
static List<TableList.Tables> listTables(Bigquery bq, String projectId, String datasetId, Predicate<TableList.Tables> needle)
        throws IOException
{
    List<TableList.Tables> tables = new ArrayList<>();
    Bigquery.Tables.List req = bq.tables().list(projectId, datasetId);
    TableList tableList;
    do {
        tableList = req.execute();
        if (tableList.getTables() != null) {
            tableList.getTables().stream().filter(needle).forEach(tables::add);
        }
        req.setPageToken(tableList.getNextPageToken());
    }
    while (null != tableList.getNextPageToken());
    return tables;
}
 

import com.google.api.services.bigquery.Bigquery; //导入依赖的package包/类
static List<DatasetList.Datasets> listDatasets(Bigquery bq, String projectId, Predicate<DatasetList.Datasets> needle)
        throws IOException
{
    List<DatasetList.Datasets> datasets = new ArrayList<>();
    Bigquery.Datasets.List req = bq.datasets().list(projectId);
    DatasetList datasetList;
    do {
        datasetList = req.execute();
        if (datasetList.getDatasets() != null) {
            datasetList.getDatasets().stream()
                    .filter(needle)
                    .forEach(datasets::add);
        }
        req.setPageToken(datasetList.getNextPageToken());
    }
    while (null != datasetList.getNextPageToken());
    return datasets;
}
 

import com.google.api.services.bigquery.Bigquery; //导入依赖的package包/类
/**
 * Creates a Query Job for a particular query on a dataset
 *
 * @param bigquery  an authorized BigQuery client
 * @param projectId a String containing the project ID
 * @param querySql  the actual query string
 * @return a reference to the inserted query job
 * @throws IOException
 */
public static JobReference startQuery(Bigquery bigquery, String projectId,
                                      String querySql) throws IOException {
    System.err.format("\nInserting Query Job: %s\n", querySql);

    Job job = new Job();
    JobConfiguration config = new JobConfiguration();
    JobConfigurationQuery queryConfig = new JobConfigurationQuery();
    config.setQuery(queryConfig);

    job.setConfiguration(config);
    queryConfig.setQuery(querySql);

    Bigquery.Jobs.Insert insert = bigquery.jobs().insert(projectId, job);
    insert.setProjectId(projectId);
    JobReference jobId = insert.execute().getJobReference();

    System.err.format("\nJob ID of Query Job is: %s\n", jobId.getJobId());

    return jobId;
}
 

import com.google.api.services.bigquery.Bigquery; //导入依赖的package包/类
/**
 * Polls the status of a BigQuery job, returns Job reference if "Done"
 *
 * @param bigquery  an authorized BigQuery client
 * @param projectId a string containing the current project ID
 * @param jobId     a reference to an inserted query Job
 * @return a reference to the completed Job
 * @throws IOException
 * @throws InterruptedException
 */
private static Job checkQueryResults(Bigquery bigquery, String projectId, JobReference jobId)
        throws IOException, InterruptedException {
    // Variables to keep track of total query time
    long startTime = System.currentTimeMillis();
    long elapsedTime;

    while (true) {
        Job pollJob = bigquery.jobs().get(projectId, jobId.getJobId()).execute();
        elapsedTime = System.currentTimeMillis() - startTime;
        System.err.format("Job status (%dms) %s: %s\n", elapsedTime,
                jobId.getJobId(), pollJob.getStatus().getState());
        if (pollJob.getStatus().getState().equals("DONE")) {
            return pollJob;
        }
        // Pause execution for one second before polling job status again, to
        // reduce unnecessary calls to the BigQUery API and lower overall
        // application bandwidth.
        Thread.sleep(1000);
    }
}