Java BlockingDataflowPipelineRunner类代码示例

import com.google.cloud.dataflow.sdk.runners.BlockingDataflowPipelineRunner; //导入依赖的package包/类
private Pipeline setupPipeline(final String inputPath, final String outputPath, boolean enableGcs, boolean enableCloudExec) {
    final GATKGCSOptions options = PipelineOptionsFactory.as(GATKGCSOptions.class);
    if (enableCloudExec) {
        options.setStagingLocation(getGCPTestStaging());
        options.setProject(getGCPTestProject());
        options.setRunner(BlockingDataflowPipelineRunner.class);
    } else if (BucketUtils.isHadoopUrl(inputPath) || BucketUtils.isHadoopUrl(outputPath)) {
        options.setRunner(SparkPipelineRunner.class);
    } else {
        options.setRunner(DirectPipelineRunner.class);
    }
    if (enableGcs) {
        options.setApiKey(getGCPTestApiKey());
    }
    final Pipeline p = Pipeline.create(options);
    DataflowUtils.registerGATKCoders(p);
    return p;
}
 

import com.google.cloud.dataflow.sdk.runners.BlockingDataflowPipelineRunner; //导入依赖的package包/类
private PipelineOptions getCloudExecutionOptions(String stagingLocation) {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setProject(Constants.PROJECT_ID);
  options.setStagingLocation(stagingLocation);
  options.setRunner(BlockingDataflowPipelineRunner.class);
  return options;
}
 

import com.google.cloud.dataflow.sdk.runners.BlockingDataflowPipelineRunner; //导入依赖的package包/类
private PipelineOptions getCloudExecutionOptions(String stagingLocation) {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setProject(SystemProperty.applicationId.get());
  options.setStagingLocation(stagingLocation);
  options.setRunner(BlockingDataflowPipelineRunner.class);
  return options;
}
 

import com.google.cloud.dataflow.sdk.runners.BlockingDataflowPipelineRunner; //导入依赖的package包/类
private static PipelineOptions getCloudExecutionOptions(String stagingLocation) {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setProject(SystemProperty.applicationId.get());
  options.setStagingLocation(stagingLocation);
  options.setRunner(BlockingDataflowPipelineRunner.class);
  return options;
}
 

import com.google.cloud.dataflow.sdk.runners.BlockingDataflowPipelineRunner; //导入依赖的package包/类
private static Class<? extends PipelineRunner<?>> runner(String name) {
  Class<? extends PipelineRunner<?>> c = DirectPipelineRunner.class; // default

  if (DEFAULT_RUNNER.equals(name) || name == null) {
    c = DataflowPipelineRunner.class;
  } else if (BLOCKING_RUNNER.equals(name)) {
    c = BlockingDataflowPipelineRunner.class;
  } else if (DIRECT_RUNNER.equals(name)) {
    c = DirectPipelineRunner.class;
  }
  return c;
}
 

import com.google.cloud.dataflow.sdk.runners.BlockingDataflowPipelineRunner; //导入依赖的package包/类
public static void run() {
  DataflowPipelineOptions options = PipelineOptionsFactory.create()
      .as(DataflowPipelineOptions.class);
  options.setRunner(BlockingDataflowPipelineRunner.class);
  options.setProject("chrome-oven-144308");
  options.setFilesToStage(
      detectClassPathResourcesToStage(
          DataflowPipelineRunner.class.getClassLoader()
      )
  );
  options.setStagingLocation("gs://dataflow-chrome-oven-144308/stagingForScheduledPipeline");

  Pipeline p = Pipeline.create(options);

  System.out.println("get here 0");
  p.apply(TextIO.Read.from("gs://dataflow-samples/shakespeare/*"))
      .apply(ParDo.named("ExtractWords").of(new DoFn<String, String>() {
        @Override
        public void processElement(ProcessContext c) {
          System.out.println("get here 1");
          for (String word : c.element().split("[^a-zA-Z']+")) {
            if (!word.isEmpty()) {
              c.output(word);
            }
          }
        }
      }))
      .apply(Count.<String>perElement())
      .apply("FormatResults", MapElements.via(new SimpleFunction<KV<String, Long>, String>() {
        @Override
        public String apply(KV<String, Long> input) {
          System.out.println("get here 3");
          return input.getKey() + ": " + input.getValue();
        }
      }))

      .apply(TextIO.Write.to("gs://dataflow-chrome-oven-144308/scheduled"));

  p.run();
}
 

import com.google.cloud.dataflow.sdk.runners.BlockingDataflowPipelineRunner; //导入依赖的package包/类
public static void run(String[] args) {
  System.out.println("Making GCS->Datastore pipeline");

  Options options = PipelineOptionsFactory.fromArgs(args)
      .withValidation()
      .as(Options.class);

  if (options.getIsBlocking()) {
    options.setRunner(BlockingDataflowPipelineRunner.class);
  } else {
    options.setRunner(DataflowPipelineRunner.class);
    options.setStreaming(false);
  }

  // Create Datastore sink
  DatastoreV1.Write write = DatastoreIO.v1().write()
      .withProjectId(options.getProject());


  // Build our data pipeline
  Pipeline pipeline = Pipeline.create(options);
  pipeline.apply("ReadBackup", TextIO.Read.from(options.getBackupGCSPrefix() + "*"))
  .apply("JsonToEntity", ParDo.of(new JsonToEntity()))
  .apply("EntityToDatastore", write);

  System.out.println("Running pipeline");
  pipeline.run();
}
 

import com.google.cloud.dataflow.sdk.runners.BlockingDataflowPipelineRunner; //导入依赖的package包/类
public static void run(String[] args) {
  System.out.println("Making Datastore->GCS pipeline");

  Options options = PipelineOptionsFactory.fromArgs(args)
      .withValidation()
      .as(Options.class);

  if (options.getIsBlocking()) {
    options.setRunner(BlockingDataflowPipelineRunner.class);
  } else {
    options.setRunner(DataflowPipelineRunner.class);
    options.setStreaming(false);
  }


  // Add Timestamp + Entity Kind to backup files
  DateFormat dateFormat = new SimpleDateFormat("yyyy.MM.dd_HH.mm.ss");
  String outputLocation = String.format("%s.%s",
      options.getDatastoreEntityKind() ,
      dateFormat.format(new Date()));

  if (options.getBackupGCSPrefix().endsWith("/")) {
    outputLocation = options.getBackupGCSPrefix() + outputLocation + "/";
  } else {
    outputLocation = options.getBackupGCSPrefix() + "." + outputLocation + "/";
  }

  // Build our Datastore query.
  // Right now we are simply grabbing all Datastore records of a given kind,
  Query.Builder queryBuilder = Query.newBuilder();
  queryBuilder.addKindBuilder().setName(options.getDatastoreEntityKind());
  Query query = queryBuilder.build();

  // Generate the Datastore Read Source
  DatastoreV1.Read read = DatastoreIO.v1().read()
      .withProjectId(options.getProject())
      .withQuery(query);

  // Build our data pipeline
  Pipeline pipeline = Pipeline.create(options);
  pipeline.apply("IngestEntities", read)
          .apply("EntityToJson", ParDo.of(new DatastoreToJson()))
          .apply("WriteJson", TextIO.Write.to(outputLocation)
              .withSuffix(".json"));

  System.out.println("Running pipeline");
  pipeline.run();
}
 

import com.google.cloud.dataflow.sdk.runners.BlockingDataflowPipelineRunner; //导入依赖的package包/类
public static void run(String[] args) throws Exception {
  System.out.println("Making Datastore->GCS pipeline");

  Options options = PipelineOptionsFactory.fromArgs(args)
      .withValidation()
      .as(Options.class);

  if (options.getIsBlocking()) {
    options.setRunner(BlockingDataflowPipelineRunner.class);
  } else {
    options.setRunner(DataflowPipelineRunner.class);
    options.setStreaming(false);
  }

  String date = (new SimpleDateFormat("yyyMMdd")).format(new Date());

  // make string in format: "my-project:dataset.entity_date"
  String tableName = String.format("%s:%s.%s_%s",
      options.getProject(),
      options.getBQDataset(),
      options.getDatastoreEntityKind(),
      date);
  System.out.println("Destination BigQuery Table is: " + tableName);

  // Build our Datastore query.
  // Right now we are simply grabbing all Datastore records of a given kind,
  Query.Builder queryBuilder = Query.newBuilder();
  queryBuilder.addKindBuilder().setName(options.getDatastoreEntityKind());
  Query query = queryBuilder.build();

  // Generate the Datastore Read Source
  DatastoreV1.Read read = DatastoreIO.v1().read()
      .withProjectId(options.getProject())
      .withQuery(query);
  
  // Build our data pipeline
  Pipeline pipeline = Pipeline.create(options);
  pipeline.apply("IngestEntities", read)
          .apply("EntityToTableRows", ParDo.of(new DatastoreToTableRow()))
          .apply("WriteTableRows", BigQueryIO.Write
              .to(tableName)
              .withSchema(new BQSchema().getTableSchema())
              .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_TRUNCATE)
              .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED));

  System.out.println("Running pipeline");
  pipeline.run();
}