当前位置: 首页>>代码示例>>Java>>正文


Java PipelineOptionsValidator.validate方法代码示例

本文整理汇总了Java中org.apache.beam.sdk.options.PipelineOptionsValidator.validate方法的典型用法代码示例。如果您正苦于以下问题:Java PipelineOptionsValidator.validate方法的具体用法?Java PipelineOptionsValidator.validate怎么用?Java PipelineOptionsValidator.validate使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在org.apache.beam.sdk.options.PipelineOptionsValidator的用法示例。


在下文中一共展示了PipelineOptionsValidator.validate方法的12个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: fromOptions

import org.apache.beam.sdk.options.PipelineOptionsValidator; //导入方法依赖的package包/类
/**
 * Creates and returns a new SamzaRunner with provided options.
 *
 * @param options The PipelineOptions to use when executing the job.
 * @return A pipeline runner that will execute with specified options.
 */
public static SamzaRunner fromOptions(PipelineOptions options) {
  SamzaPipelineOptions samzaOptions = PipelineOptionsValidator.validate(SamzaPipelineOptions.class, options);

  ArrayList<String> missing = new ArrayList<>();

  if (samzaOptions.getAppName() == null) {
    missing.add("appName");
  }
  if (missing.size() > 0) {
    throw new IllegalArgumentException(
        "Missing required values: " + Joiner.on(',').join(missing));
  }

  return new SamzaRunner(samzaOptions);
}
 
开发者ID:FreshetDMS,项目名称:Freshet-Deprecated,代码行数:22,代码来源:SamzaRunner.java

示例2: fromOptions

import org.apache.beam.sdk.options.PipelineOptionsValidator; //导入方法依赖的package包/类
/**
 * Constructs a runner from the provided {@link PipelineOptions}.
 *
 * @return The newly created runner.
 */
public static PipelineRunner<? extends PipelineResult> fromOptions(PipelineOptions options) {
  checkNotNull(options);
  PipelineOptionsValidator.validate(PipelineOptions.class, options);

  // (Re-)register standard FileSystems. Clobbers any prior credentials.
  FileSystems.setDefaultPipelineOptions(options);

  @SuppressWarnings("unchecked")
  PipelineRunner<? extends PipelineResult> result =
      InstanceBuilder.ofType(PipelineRunner.class)
      .fromClass(options.getRunner())
      .fromFactoryMethod("fromOptions")
      .withArg(PipelineOptions.class, options)
      .build();
  return result;
}
 
开发者ID:apache,项目名称:beam,代码行数:22,代码来源:PipelineRunner.java

示例3: fromOptions

import org.apache.beam.sdk.options.PipelineOptionsValidator; //导入方法依赖的package包/类
/**
 * Creates and returns a new SparkRunner with specified options.
 *
 * @param options The PipelineOptions to use when executing the job.
 * @return A pipeline runner that will execute with specified options.
 */
public static SparkRunner fromOptions(PipelineOptions options) {
  SparkPipelineOptions sparkOptions =
      PipelineOptionsValidator.validate(SparkPipelineOptions.class, options);

  if (sparkOptions.getFilesToStage() == null) {
    sparkOptions.setFilesToStage(detectClassPathResourcesToStage(
        SparkRunner.class.getClassLoader()));
    LOG.info("PipelineOptions.filesToStage was not specified. "
            + "Defaulting to files from the classpath: will stage {} files. "
            + "Enable logging at DEBUG level to see which files will be staged.",
        sparkOptions.getFilesToStage().size());
    LOG.debug("Classpath elements: {}", sparkOptions.getFilesToStage());
  }

  return new SparkRunner(sparkOptions);
}
 
开发者ID:apache,项目名称:beam,代码行数:23,代码来源:SparkRunner.java

示例4: readTestPipelineOptions

import org.apache.beam.sdk.options.PipelineOptionsValidator; //导入方法依赖的package包/类
public static IOTestPipelineOptions readTestPipelineOptions() {
  PipelineOptionsFactory.register(IOTestPipelineOptions.class);
  IOTestPipelineOptions options = TestPipeline
      .testingPipelineOptions()
      .as(IOTestPipelineOptions.class);

  return PipelineOptionsValidator.validate(IOTestPipelineOptions.class, options);
}
 
开发者ID:apache,项目名称:beam,代码行数:9,代码来源:FileBasedIOITHelper.java

示例5: fromOptions

import org.apache.beam.sdk.options.PipelineOptionsValidator; //导入方法依赖的package包/类
public static SparkRunnerDebugger fromOptions(PipelineOptions options) {
  if (options instanceof TestSparkPipelineOptions) {
    TestSparkPipelineOptions testSparkPipelineOptions =
        PipelineOptionsValidator.validate(TestSparkPipelineOptions.class, options);
    return new SparkRunnerDebugger(testSparkPipelineOptions);
  } else {
    SparkPipelineOptions sparkPipelineOptions =
        PipelineOptionsValidator.validate(SparkPipelineOptions.class, options);
    return new SparkRunnerDebugger(sparkPipelineOptions);
  }
}
 
开发者ID:apache,项目名称:beam,代码行数:12,代码来源:SparkRunnerDebugger.java

示例6: fromOptions

import org.apache.beam.sdk.options.PipelineOptionsValidator; //导入方法依赖的package包/类
/**
 * Construct a runner from the provided options.
 *
 * @param options Properties which configure the runner.
 * @return The newly created runner.
 */
public static FlinkRunner fromOptions(PipelineOptions options) {
  FlinkPipelineOptions flinkOptions =
      PipelineOptionsValidator.validate(FlinkPipelineOptions.class, options);
  ArrayList<String> missing = new ArrayList<>();

  if (flinkOptions.getAppName() == null) {
    missing.add("appName");
  }
  if (missing.size() > 0) {
    throw new IllegalArgumentException(
        "Missing required values: " + Joiner.on(',').join(missing));
  }

  if (flinkOptions.getFilesToStage() == null) {
    flinkOptions.setFilesToStage(detectClassPathResourcesToStage(
        FlinkRunner.class.getClassLoader()));
    LOG.info("PipelineOptions.filesToStage was not specified. "
            + "Defaulting to files from the classpath: will stage {} files. "
            + "Enable logging at DEBUG level to see which files will be staged.",
        flinkOptions.getFilesToStage().size());
    LOG.debug("Classpath elements: {}", flinkOptions.getFilesToStage());
  }

  // Set Flink Master to [auto] if no option was specified.
  if (flinkOptions.getFlinkMaster() == null) {
    flinkOptions.setFlinkMaster("[auto]");
  }

  return new FlinkRunner(flinkOptions);
}
 
开发者ID:apache,项目名称:beam,代码行数:37,代码来源:FlinkRunner.java

示例7: fromOptions

import org.apache.beam.sdk.options.PipelineOptionsValidator; //导入方法依赖的package包/类
public static ApexRunner fromOptions(PipelineOptions options) {
  ApexPipelineOptions apexPipelineOptions =
          PipelineOptionsValidator.validate(ApexPipelineOptions.class, options);
  return new ApexRunner(apexPipelineOptions);
}
 
开发者ID:apache,项目名称:beam,代码行数:6,代码来源:ApexRunner.java

示例8: fromOptions

import org.apache.beam.sdk.options.PipelineOptionsValidator; //导入方法依赖的package包/类
public static TestApexRunner fromOptions(PipelineOptions options) {
  ApexPipelineOptions apexOptions = PipelineOptionsValidator
      .validate(ApexPipelineOptions.class, options);
  return new TestApexRunner(apexOptions);
}
 
开发者ID:apache,项目名称:beam,代码行数:6,代码来源:TestApexRunner.java

示例9: fromOptions

import org.apache.beam.sdk.options.PipelineOptionsValidator; //导入方法依赖的package包/类
public static TestFlinkRunner fromOptions(PipelineOptions options) {
  FlinkPipelineOptions flinkOptions =
      PipelineOptionsValidator.validate(FlinkPipelineOptions.class, options);
  return new TestFlinkRunner(flinkOptions);
}
 
开发者ID:apache,项目名称:beam,代码行数:6,代码来源:TestFlinkRunner.java

示例10: fromOptions

import org.apache.beam.sdk.options.PipelineOptionsValidator; //导入方法依赖的package包/类
public static GearpumpRunner fromOptions(PipelineOptions options) {
  GearpumpPipelineOptions pipelineOptions =
      PipelineOptionsValidator.validate(GearpumpPipelineOptions.class, options);
  return new GearpumpRunner(pipelineOptions);
}
 
开发者ID:apache,项目名称:beam,代码行数:6,代码来源:GearpumpRunner.java

示例11: fromOptions

import org.apache.beam.sdk.options.PipelineOptionsValidator; //导入方法依赖的package包/类
public static TestGearpumpRunner fromOptions(PipelineOptions options) {
  GearpumpPipelineOptions pipelineOptions =
      PipelineOptionsValidator.validate(GearpumpPipelineOptions.class, options);
  return new TestGearpumpRunner(pipelineOptions);
}
 
开发者ID:apache,项目名称:beam,代码行数:6,代码来源:TestGearpumpRunner.java

示例12: populateDAG

import org.apache.beam.sdk.options.PipelineOptionsValidator; //导入方法依赖的package包/类
@Override
  public void populateDAG(DAG dag, Configuration conf)
  {
/*
    HadoopFileSystemOptions fsoptions = PipelineOptionsFactory.as(HadoopFileSystemOptions.class);
    //fsoptions.setHdfsConfiguration(ImmutableList.of(new Configuration()));
    for (FileSystemRegistrar registrar
        : Lists.newArrayList(ServiceLoader.load(FileSystemRegistrar.class).iterator())) {
      System.out.println(registrar);
      if (registrar instanceof HadoopFileSystemRegistrar) {
        Iterable<FileSystem> fileSystems = registrar.fromOptions(fsoptions);
        for (FileSystem fs : fileSystems) {
          System.out.println(fs);
        }
      }
    }
*/
    String optionsStr = conf.get(KEY_PIPELINE_OPTIONS, "--runner=ApexRunner");
    String[] args = StringUtils.splitByWholeSeparator(optionsStr, " ");
    WordCountOptions options = PipelineOptionsFactory.fromArgs(args).withValidation()
        .as(WordCountOptions.class);
    Pipeline p = Pipeline.create(options);
    // Concepts #2 and #3: Our pipeline applies the composite CountWords transform, and passes the
    // static FormatAsTextFn() to the ParDo transform.
      p.apply("ReadFromHDFS", TextIO.read().from(absoluteUri(options.getInputFile())))
      .apply(new CountWords())
      .apply(MapElements.via(new FormatAsTextFn()))
      .apply("WriteToHDFS", TextIO.write().to(absoluteUri(options.getOutput())).withNumShards(2));

    ApexPipelineOptions apexPipelineOptions =
        PipelineOptionsValidator.validate(ApexPipelineOptions.class, options);
    final ApexPipelineTranslator translator = new ApexPipelineTranslator(apexPipelineOptions);

    // roundabout way to apply overrides - we just want to translate, not run the pipeline here
    ApexRunner runner = new ApexRunner(apexPipelineOptions);
    try {
      Method m = ApexRunner.class.getDeclaredMethod("getOverrides");
      m.setAccessible(true);
      @SuppressWarnings("unchecked")
      List<PTransformOverride> overrides = (List<PTransformOverride>)m.invoke(runner);
      p.replaceAll(overrides);
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
    translator.translate(p, dag);
  }
 
开发者ID:tweise,项目名称:apex-samples,代码行数:47,代码来源:Application.java


注:本文中的org.apache.beam.sdk.options.PipelineOptionsValidator.validate方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。