Java TableRow类代码示例

import com.google.api.services.bigquery.model.TableRow; //导入依赖的package包/类
private TableRow toTableRow(List<TableFieldSchema> tableSchema, Entity e) throws IOException {

    TableRow row = new TableRow();
    Map<String, Value> fields = e.getPropertiesMap();

    for (TableFieldSchema columnSchema : tableSchema) {
      if (columnSchema.getName().equals("__key__") && columnSchema.getType().toUpperCase().equals("STRING")) {
        row.put(columnSchema.getName(), keyToString(e.getKey()));
      }
      else if (fields.containsKey(columnSchema.getName())) {
        Value v = fields.get(columnSchema.getName());
        row.put(columnSchema.getName(), toRowValue(columnSchema, v));
      }
    }
    return row;
  }
 

import com.google.api.services.bigquery.model.TableRow; //导入依赖的package包/类
public static void main(String[] args) {
	BigQueryToDatastoreOptions options = PipelineOptionsFactory.fromArgs(args).withValidation()
			.as(BigQueryToDatastoreOptions.class);

	String inputTable = options.getInputTable().get();
	String projectID = options.getOutputProjectID().get();
	String kind = options.getOutputKind().get();

	LOG.info("Input_Table : " + inputTable);
	LOG.info("ProjectID : " + projectID);
	LOG.info("Kind : " + kind);

	Pipeline p = Pipeline.create(options);

	PCollection<KV<Integer, Iterable<TableRow>>> keywordGroups = p
			.apply(BigQueryIO.Read.named("ReadUtterance").from(inputTable)).apply(new GroupKeywords());
	
	CreateEntities createEntities = new CreateEntities();
	createEntities.setKind(kind);
	
	PCollection<Entity> entities = keywordGroups.apply(createEntities);
	entities.apply(DatastoreIO.v1().write().withProjectId(projectID));

	p.run();
}
 

import com.google.api.services.bigquery.model.TableRow; //导入依赖的package包/类
public static String[] extractRedditPostMetaFields(TableRow post) {
	String[] result = new String[METAFIELDS_REDDIT_NUM_FIELDS];
	
	String domain = post.get("domain").toString();
	if (!domain.startsWith("self.")) {
		result[METAFIELDS_REDDIT_EXTERNALLINK_IDX] = post.get("url").toString();
		result[METAFIELDS_REDDIT_DOMAIN_IDX] = domain;
	} else {
		result[METAFIELDS_REDDIT_EXTERNALLINK_IDX] = METAFIELDS_VALUE_UNVAILABLE;
		result[METAFIELDS_REDDIT_DOMAIN_IDX] = METAFIELDS_VALUE_UNVAILABLE;
	}
	Object oSubreddit = post.get("subreddit");
	if (oSubreddit != null)
		result[METAFIELDS_REDDIT_SUBREDDIT_IDX] = oSubreddit.toString();
	else 
		result[METAFIELDS_REDDIT_SUBREDDIT_IDX] = METAFIELDS_VALUE_UNVAILABLE;
		
	result[METAFIELDS_REDDIT_SCORE_IDX] = post.get("score").toString();
	result[METAFIELDS_REDDIT_POSTID_IDX] = extractPostIdFromRedditPost(post);
	
	return result;
}
 

import com.google.api.services.bigquery.model.TableRow; //导入依赖的package包/类
@ProcessElement
public void processElement(ProcessContext c) {
	WebresourceSocialCount sc = c.element();
	
	Instant countTime = new Instant(sc.countTime);
	
	TableRow row = new TableRow()
		.set("WebResourceHash", sc.webResourceHash)
		.set("WrPublicationDateId", sc.wrPublicationDateId)
		.set("CountTime", countTime.toString())
		.set("DocumentCollectionId", sc.documentCollectionId)
		.set("CollectionItemId", sc.collectionItemId)
		.set("FbCount", sc.fbCount)
		.set("TwCount", sc.twCount);

	c.output(row);

}
 

import com.google.api.services.bigquery.model.TableRow; //导入依赖的package包/类
/**
 * @param Document indexes
 * @return a POJO containing 2 PCollections: Unique docs, and Duplicates
 */
private static ContentDuplicateOrNot filterSoftDuplicates(
		PCollection<ContentIndexSummary> indexes) {
	// 
	PCollectionTuple dedupeOrNot = indexes
		.apply("Extract Text grouping key", 
			ParDo.of(new GetContentIndexSummaryKeyFn()))
		.apply("Group by Text grouping key", 
			GroupByKey.<ContentSoftDeduplicationKey, ContentIndexSummary>create())
		.apply("Eliminate Text dupes", 
			ParDo.of(new EliminateTextDupes())
				.withOutputTags(PipelineTags.indexedContentNotToDedupeTag, 
					TupleTagList.of(PipelineTags.indexedContentToDedupeTag))); 	
		
	PCollection<TableRow> dedupedWebresources = 
		dedupeOrNot.get(PipelineTags.indexedContentToDedupeTag)
			.apply(ParDo.of(new CreateWebresourceTableRowFromDupeIndexSummaryFn()));
	
	ContentDuplicateOrNot contentDuplicateOrNot = new ContentDuplicateOrNot(
		dedupeOrNot.get(PipelineTags.indexedContentNotToDedupeTag),
		dedupedWebresources);
	
	return contentDuplicateOrNot;
}
 

import com.google.api.services.bigquery.model.TableRow; //导入依赖的package包/类
/**
 * input - a tupel that contains the data element (TableRow), the window, the timestamp, and the pane
 */

@Override
public TableDestination apply(ValueInSingleWindow<TableRow> input) {
    
	String partition;
	
	if (this.isTimeField) {
     String sTime = (String) input.getValue().get(this.fieldName);
     Instant time = Instant.parse(sTime);
     partition = time.toString(partitionFormatter);
	} else {
		partition = ((Integer) input.getValue().get(this.fieldName)).toString();
	}
	
    TableReference reference = new TableReference();
    reference.setProjectId(this.projectId);
    reference.setDatasetId(this.datasetId);
    reference.setTableId(this.partitionPrefix + partition);
    return new TableDestination(reference, null);
}
 

import com.google.api.services.bigquery.model.TableRow; //导入依赖的package包/类
@ProcessElement
public void processElement(ProcessContext c) throws Exception {
  Entity entity = c.element();
  EntityBQTransform ebt = EntityBQTransform.newBuilder()
      .setRowSchema(tableSchema().getFields())
      .setStrictCast(strictCast().get())
      .build();

  TableRow row = ebt.toTableRow(entity);

  if (jsTransform().hasTransform()) {
    String rowJson = jsTransform().invoke(
        Transport.getJsonFactory().toString(entity),
        Transport.getJsonFactory().toString(row));
    row = Transport.getJsonFactory().fromString(rowJson, TableRow.class);
  }

  c.output(row);
}
 

import com.google.api.services.bigquery.model.TableRow; //导入依赖的package包/类
@Test
public void testDatastoreToBq_EntityToTableRow_notransform() throws Exception, IOException {
  DoFnTester<Entity, TableRow> fnTester = DoFnTester.of(EntityToTableRow.newBuilder()
      .setStrictCast(StaticValueProvider.of(true))
      .setTableSchemaJson(StaticValueProvider.of(mTableSchemaJson))
      .setJsTransformFunctionName(StaticValueProvider.of(null))
      .setJsTransformPath(StaticValueProvider.of(null))
  .build());

  Builder entityBuilder = Entity.newBuilder();
  JsonFormat.parser().usingTypeRegistry(
      TypeRegistry.newBuilder()
          .add(Entity.getDescriptor())
          .build())
      .merge(mEntityJson, entityBuilder);

  Entity entity = entityBuilder.build();
  List<TableRow> tableRows = fnTester.processBundle(entity);
  TableRow tr = tableRows.get(0);

  Assert.assertEquals(1, tableRows.size());
  Assert.assertEquals("key(Drawing, '31ce830e-91d0-405e-855a-abe416cadc1f')", tr.get("__key__"));
  Assert.assertEquals("79a1d9d9-e255-427a-9b09-f45157e97790", tr.get("canvasId"));
}
 

import com.google.api.services.bigquery.model.TableRow; //导入依赖的package包/类
@Override
public void processElement(ProcessContext c) {
  Double dollars = c.element();
  TableRow r = new TableRow();
  r.set("dollar_turnover", dollars);
  // the timing can be:
  // EARLY: the dollar amount is not yet final
  // ON_TIME: dataflow thinks the dollar amount is final but late data are still possible
  // LATE: late data has arrived
  r.set("dollar_timing", c.pane().getTiming()); // EARLY, ON_TIME or LATE
  r.set("dollar_window", ((IntervalWindow) c.window()).start().getMillis() / 1000.0 / 60.0); // timestamp in fractional minutes

  LOG.info("Outputting $ value {}} at {} with marker {} for window {}",
    dollars.toString(), new Date().getTime(), c.pane().getTiming().toString(), c.window().hashCode());
  c.output(r);
}
 

import com.google.api.services.bigquery.model.TableRow; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
    PipelineOptionsFactory.register(TemplateOptions.class);
    TemplateOptions options = PipelineOptionsFactory
            .fromArgs(args)
            .withValidation()
            .as(TemplateOptions.class);
    options.setAutoscalingAlgorithm(THROUGHPUT_BASED);
    Pipeline pipeline = Pipeline.create(options);
    pipeline.apply(BigQueryIO.read().from(options.getBigQueryTableName()))
            .apply(ParDo.of(new DoFn<TableRow, String>() {
                @ProcessElement
                public void processElement(ProcessContext c) throws Exception {
                    String commaSep = c.element().values()
                            .stream()
                            .map(cell -> cell.toString().trim())
                            .collect(Collectors.joining("\",\""));
                    c.output(commaSep);
                }
            }))
            .apply(TextIO.write().to(options.getOutputFile())
                    .withoutSharding()
                    .withWritableByteChannelFactory(GZIP)
            );
    pipeline.run();
}
 

import com.google.api.services.bigquery.model.TableRow; //导入依赖的package包/类
@Test
public void testApply() {
  ObjectMapper mapperObj = new ObjectMapper();
  Map map = new HashMap();
  map.put("foo", 1);
  map.put("bar", Arrays.asList(1, 2, 3));
  Map submap = new HashMap();
  submap.put("moge", 1);
  map.put("hoge", submap);
  String jsonStr = new Gson().toJson(map);

  PubsubMessage2TableRowFn fn = new PubsubMessage2TableRowFn();
  DoFnTester<String, TableRow> fnTester = DoFnTester.of(fn);

  try {
    List<TableRow> result = fnTester.processBundle(jsonStr);
    String msg = result.get(0).get(PubsubMessage2TableRowFn.getMessageColumnName()).toString();
    assertEquals(jsonStr, msg);
  } catch (Exception e) {
    e.printStackTrace();
  }
}
 

import com.google.api.services.bigquery.model.TableRow; //导入依赖的package包/类
@Test
public void testTransformer() {
  // Read file
  InputStream inputStream =
      this.getClass().getResourceAsStream("/dummy-log");
  // Convert to JSON string
  String json = readStream(inputStream);

  LogAggregator.LogTransformer transformer = new LogAggregator.LogTransformer();

  List<String> inputs = Arrays.asList(json, json, json);
  PCollection<String> inputCollection = pipeline.apply(Create.of(inputs));
  PCollection<TableRow> outputCollection = inputCollection.apply(transformer);
  PAssert.that(outputCollection).satisfies(new HasColumnsCheckerFn());
  pipeline.run().waitUntilFinish();
}
 

import com.google.api.services.bigquery.model.TableRow; //导入依赖的package包/类
private long writeRows(String tableId, List<TableRow> rows, TableSchema schema,
                       String destinationPattern) throws IOException {
  Schema avroSchema = BigQueryAvroUtils.toGenericAvroSchema(tableId, schema.getFields());
  List<TableRow> rowsToWrite = Lists.newArrayList();
  int shard = 0;
  for (int i = 0; i < rows.size(); ++i) {
    rowsToWrite.add(rows.get(i));
    if (rowsToWrite.size() == 5) {
      writeRowsHelper(rowsToWrite, avroSchema, destinationPattern, shard++);
      rowsToWrite.clear();
    }
  }
  if (!rowsToWrite.isEmpty()) {
    writeRowsHelper(rowsToWrite, avroSchema, destinationPattern, shard++);
  }
  return shard;
}
 

import com.google.api.services.bigquery.model.TableRow; //导入依赖的package包/类
private boolean validateDispositions(Table table, CreateDisposition createDisposition,
                                     WriteDisposition writeDisposition)
    throws InterruptedException, IOException {
  if (table == null) {
    if (createDisposition == CreateDisposition.CREATE_NEVER) {
      return false;
    }
  } else if (writeDisposition == WriteDisposition.WRITE_TRUNCATE) {
    datasetService.deleteTable(table.getTableReference());
  } else if (writeDisposition == WriteDisposition.WRITE_EMPTY) {
    List<TableRow> allRows = datasetService.getAllRows(table.getTableReference().getProjectId(),
        table.getTableReference().getDatasetId(), table.getTableReference().getTableId());
    if (!allRows.isEmpty()) {
      return false;
    }
  }
  return true;
}
 

import com.google.api.services.bigquery.model.TableRow; //导入依赖的package包/类
@Override
public TableRow next() {
	if (nextLine == null) {
		throw new NoSuchElementException();
	}
	String result = nextLine;
	List<TableCell> cellList = new ArrayList<TableCell>();
	Matcher matcher = CVS_ELEMENT_PATTERN.matcher(result);
	while (matcher.find()) {
		TableCell resultCell = new TableCell();
		resultCell.setV(matcher.group());
		cellList.add(resultCell);
	}
	TableRow resultRow = new TableRow();
	resultRow.setF(cellList);
	// For next
	fetch();
	return resultRow;
}