本文整理汇总了Java中com.google.cloud.dataflow.sdk.io.Read类的典型用法代码示例。如果您正苦于以下问题:Java Read类的具体用法?Java Read怎么用?Java Read使用的例子?那么, 这里精选的类代码示例或许可以为您提供帮助。
Read类属于com.google.cloud.dataflow.sdk.io包,在下文中一共展示了Read类的5个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: test_reading
import com.google.cloud.dataflow.sdk.io.Read; //导入依赖的package包/类
@Test
public void test_reading() throws Exception {
final File file =
new File(getClass().getResource("/sample.csv").toURI());
assertThat(file.exists()).isTrue();
final Pipeline pipeline = TestPipeline.create();
final PCollection<String> output =
pipeline.apply(Read.from(CsvWithHeaderFileSource.from(file.getAbsolutePath())));
DataflowAssert
.that(output)
.containsInAnyOrder("a:1, b:2, c:3, ", "a:4, b:5, c:6, ");
pipeline.run();
}
示例2: constructPipeline
import com.google.cloud.dataflow.sdk.io.Read; //导入依赖的package包/类
private PCollection<String> constructPipeline(Pipeline pipeline, String org) {
// Read projects from the CRM API.
PCollection<GCPProject> projects =
pipeline.apply(Read.from(new LiveProjectSource(org)));
// List the service accounts of the projects.
PCollection<GCPServiceAccount> serviceAccounts =
projects.apply(ParDo.named("List Service Accounts").of(new ListServiceAccounts()));
// List the keys of the service accounts.
PCollection<GCPServiceAccountKey> serviceAccountKeys =
serviceAccounts.apply(ParDo.named("List Service Account Keys")
.of(new ListServiceAccountKeys()));
// Construct an alert message for all the discrepancies found.
return serviceAccountKeys.apply(ParDo
.named("Remove user-managed keys")
.of(new ExportedServiceAccountKeyMessenger()));
}
示例3: main
import com.google.cloud.dataflow.sdk.io.Read; //导入依赖的package包/类
public static void main(String[] args) {
CloudBigtableOptions options =
PipelineOptionsFactory.fromArgs(args).withValidation().as(CloudBigtableOptions.class);
CloudBigtableScanConfiguration config =
new CloudBigtableScanConfiguration.Builder()
.withProjectId(options.getBigtableProjectId())
.withInstanceId(options.getBigtableInstanceId())
.withTableId(options.getBigtableTableId())
.build();
options.setStreaming(true);
options.setRunner(DataflowPipelineRunner.class);
Pipeline p = Pipeline.create(options);
CloudBigtableIO.initializeForWrite(p);
p.apply(Read.from(new CoinbaseSource()))
.apply(ParDo.named("DeserializeCoinbase").of(new DeserializeCoinbase()))
.apply(ParDo.of(new HBaseBigtableWriter()))
.apply(CloudBigtableIO.writeToTable(config));
p.run();
}
示例4: constructPipeline
import com.google.cloud.dataflow.sdk.io.Read; //导入依赖的package包/类
private PCollection<String> constructPipeline(Pipeline pipeline, String org,
BoundedSource<KV<List<String>, String>> knownGoodSource) {
// Read files from GCS.
PCollection<KV<List<String>, String>> knownGoodFiles =
pipeline.apply("Read known-good data", Read.from(knownGoodSource));
// Convert files to GCPResourceState objects.
PCollection<KV<GCPResource, GCPResourceState>> knownGoodStates =
knownGoodFiles.apply(ParDo.named("Convert file data to Java Objects")
.of(new FileToState()));
// Tag the state objects to indicate they're from a checked-in repo and not live.
PCollection<KV<GCPResource, KV<StateSource, GCPResourceState>>> taggedKnownGoodStates =
knownGoodStates.apply(ParDo.named("Mark states as being known-good")
.of(new TagStateWithSource(StateSource.DESIRED)));
// Read projects from the CRM API.
PCollection<GCPProject> allProjects =
pipeline.apply("Read live projects", Read.from(new LiveProjectSource(org)));
// Extract project states.
PCollection<KV<GCPResource, GCPResourceState>> liveStates =
allProjects
.apply(ParDo.named("Extract project policies").of(new ExtractState()));
// Tag the states to indicate they're live and not from a checked-in source.
PCollection<KV<GCPResource, KV<StateSource, GCPResourceState>>> taggedLiveStates =
liveStates.apply(ParDo.named("Mark states as being live")
.of(new TagStateWithSource(StateSource.LIVE)));
// Join the two known-good and the live halves.
PCollectionView<Map<GCPResource, KV<StateSource, GCPResourceState>>> knownGoodStatesView =
taggedKnownGoodStates.apply(View.<GCPResource, KV<StateSource, GCPResourceState>>asMap());
PCollection<KV<GCPResource, Map<StateSource, GCPResourceState>>> mismatchedStates =
taggedLiveStates.apply(ParDo.named("Find states that don't match")
.withSideInputs(knownGoodStatesView)
.of(new FilterOutMatchingState(knownGoodStatesView)));
// Construct an alert message for all the discrepancies found and fix the discrepancies.
return mismatchedStates
.apply(ParDo.named("Fix discrepancies").of(discrepancyAutoFixMessenger));
}
示例5: constructPipeline
import com.google.cloud.dataflow.sdk.io.Read; //导入依赖的package包/类
private PCollection<String> constructPipeline(Pipeline pipeline,
BoundedSource<KV<List<String>, String>> knownGoodSource) {
// Read files from GCS.
PCollection<KV<List<String>, String>> knownGoodFiles =
pipeline.apply("Read known-good data", Read.from(knownGoodSource));
// Convert files to GCPResourceState objects.
PCollection<KV<GCPResource, GCPResourceState>> knownGoodStates =
knownGoodFiles.apply(ParDo.named("Convert file data to Java objects")
.of(new FileToState()));
// Tag the state objects to indicate they're from a checked-in repo and not live.
PCollection<KV<GCPResource, KV<StateSource, GCPResourceState>>> taggedKnownGoodStates =
knownGoodStates.apply(ParDo.named("Mark states as being known-good")
.of(new TagStateWithSource(StateSource.DESIRED)));
// Extract a list of checked-in projects from GCS.
PCollection<List<String>> allFilePaths = knownGoodFiles
.apply("Extract just the file paths", ParDo.of(new FilePathFromPair()));
// Read the live version of the states of the checked-in projects.
PCollection<KV<GCPResource, GCPResourceState>> liveStates =
allFilePaths.apply(ParDo.named("Get live resource and states from file path")
.of(new FilePathToLiveState()));
// Tag the states to indicate they're live and not from a checked-in source.
PCollection<KV<GCPResource, KV<StateSource, GCPResourceState>>> taggedLiveStates =
liveStates.apply(ParDo.named("Mark states as being live")
.of(new TagStateWithSource(StateSource.LIVE)));
// Join the two known-good and the live halves.
PCollectionView<Map<GCPResource, KV<StateSource, GCPResourceState>>> liveStatesView =
taggedLiveStates.apply(View.<GCPResource, KV<StateSource, GCPResourceState>>asMap());
PCollection<KV<GCPResource, Map<StateSource, GCPResourceState>>> mismatchedStates =
taggedKnownGoodStates.apply(ParDo.named("Find states that don't match")
.withSideInputs(liveStatesView)
.of(new FilterOutMatchingState(liveStatesView)));
// Construct an alert message for all the discrepancies found.
return mismatchedStates.apply(ParDo
.named("Generate notification messages")
.of(new StateDiscrepancyMessenger()));
}