Java PipelineOptionsFactory.as方法代码示例

import com.google.cloud.dataflow.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
/**
 * Create Dataflow Pipeline options from the standard command-line options, "--project=",
 * "--runner=" and "--stagingLocation="
 *
 * @param args
 * @return
 * @throws IOException
 */
public static DataflowPipelineOptions pipelineOptions(String[] args) throws IOException {
  LOG.info("Set up Dataflow options");
  DataflowPipelineOptions o = PipelineOptionsFactory.as(DataflowPipelineOptions.class);

  Map<String, String> m = StringUtils.parseArgs(args);
  o.setProject(m.get(PROJECT));
  if (m.containsKey(STAGING)) {
    o.setStagingLocation(m.get(STAGING));
  } else if (m.containsKey(STAGING_LOCATION)) {
    o.setStagingLocation(m.get(STAGING_LOCATION));
  } else if (m.containsKey(WORKSPACE)) {
    o.setStagingLocation(m.get(WORKSPACE) + "/staging");
  }
  o.setRunner(runner(m.get(RUNNER)));
  o.setMaxNumWorkers(m.get(MAX_WORKERS) == null ? 1 : Integer.parseInt(m.get(MAX_WORKERS)));
  if (m.containsKey(MACHINE_TYPE)) {
    o.setWorkerMachineType(m.get(MACHINE_TYPE));
  } else {
    o.setWorkerMachineType(DEFAULT_MACHINE_TYPE);
  }
  return o;
}
 

import com.google.cloud.dataflow.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
private Pipeline setupPipeline(final String inputPath, final String outputPath, boolean enableGcs, boolean enableCloudExec) {
    final GATKGCSOptions options = PipelineOptionsFactory.as(GATKGCSOptions.class);
    if (enableCloudExec) {
        options.setStagingLocation(getGCPTestStaging());
        options.setProject(getGCPTestProject());
        options.setRunner(BlockingDataflowPipelineRunner.class);
    } else if (BucketUtils.isHadoopUrl(inputPath) || BucketUtils.isHadoopUrl(outputPath)) {
        options.setRunner(SparkPipelineRunner.class);
    } else {
        options.setRunner(DirectPipelineRunner.class);
    }
    if (enableGcs) {
        options.setApiKey(getGCPTestApiKey());
    }
    final Pipeline p = Pipeline.create(options);
    DataflowUtils.registerGATKCoders(p);
    return p;
}
 

import com.google.cloud.dataflow.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
private PipelineOptions getCloudExecutionOptions(String stagingLocation) {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setProject(Constants.PROJECT_ID);
  options.setStagingLocation(stagingLocation);
  options.setRunner(BlockingDataflowPipelineRunner.class);
  return options;
}
 

import com.google.cloud.dataflow.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
private PipelineOptions getCloudExecutionOptions(String stagingLocation) {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setProject(SystemProperty.applicationId.get());
  options.setStagingLocation(stagingLocation);
  options.setRunner(BlockingDataflowPipelineRunner.class);
  return options;
}
 

import com.google.cloud.dataflow.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
private static PipelineOptions getCloudExecutionOptions(String stagingLocation) {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setProject(SystemProperty.applicationId.get());
  options.setStagingLocation(stagingLocation);
  options.setRunner(BlockingDataflowPipelineRunner.class);
  return options;
}
 

import com.google.cloud.dataflow.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
public static Pipeline setup() {

    FXTimeSeriesPipelineOptions options =
        PipelineOptionsFactory.as(FXTimeSeriesPipelineOptions.class);


    // Setup the windowing variables for the test run
    options.setCandleResolution(BAR_SIZE_SEC);
    options.setCorrelationWindowSize(WINDOW_SIZE_SECS);
    options.setCorrelationWindowPeriod(SLIDING_WINDOW_PERIOD_SECS);

    // Set Runner to use
    options.setRunner(InProcessPipelineRunner.class);

    Pipeline pipeline = Pipeline.create(options);

    // Register all of the coders used within the Pipeline
    TimeSeriesCoders.registerCoders(pipeline);

    // Create WorkPacket SideInput

    return pipeline;
  }
 

import com.google.cloud.dataflow.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
public static SparkPipelineOptions create() {
  return PipelineOptionsFactory.as(SparkPipelineOptions.class);
}
 

import com.google.cloud.dataflow.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
public static SparkStreamingPipelineOptions create() {
  return PipelineOptionsFactory.as(SparkStreamingPipelineOptions.class);
}