Java PipelineOptionsFactory.register方法代码示例

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
    PipelineOptionsFactory.register(TemplateOptions.class);
    TemplateOptions options = PipelineOptionsFactory
            .fromArgs(args)
            .withValidation()
            .as(TemplateOptions.class);
    options.setAutoscalingAlgorithm(THROUGHPUT_BASED);
    Pipeline pipeline = Pipeline.create(options);
    pipeline.apply(BigQueryIO.read().from(options.getBigQueryTableName()))
            .apply(ParDo.of(new DoFn<TableRow, String>() {
                @ProcessElement
                public void processElement(ProcessContext c) throws Exception {
                    String commaSep = c.element().values()
                            .stream()
                            .map(cell -> cell.toString().trim())
                            .collect(Collectors.joining("\",\""));
                    c.output(commaSep);
                }
            }))
            .apply(TextIO.write().to(options.getOutputFile())
                    .withoutSharding()
                    .withWritableByteChannelFactory(GZIP)
            );
    pipeline.run();
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@Before
public void setup() throws Exception {
  PipelineOptionsFactory.register(BigtableTestOptions.class);
  options = TestPipeline.testingPipelineOptions().as(BigtableTestOptions.class);
  project = options.as(GcpOptions.class).getProject();

  bigtableOptions =
      new Builder()
          .setProjectId(project)
          .setInstanceId(options.getInstanceId())
          .setUserAgent("apache-beam-test")
          .build();

  session =
      new BigtableSession(
          bigtableOptions
              .toBuilder()
              .setCredentialOptions(
                  CredentialOptions.credential(options.as(GcpOptions.class).getGcpCredential()))
              .build());
  tableAdminClient = session.getTableAdminClient();
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@Test
public void testE2EBigtableRead() throws Exception {
  PipelineOptionsFactory.register(BigtableTestOptions.class);
  BigtableTestOptions options = TestPipeline.testingPipelineOptions()
      .as(BigtableTestOptions.class);

  String project = options.as(GcpOptions.class).getProject();

  BigtableOptions.Builder bigtableOptionsBuilder = new BigtableOptions.Builder()
      .setProjectId(project)
      .setInstanceId(options.getInstanceId());

  final String tableId = "BigtableReadTest";
  final long numRows = 1000L;

  Pipeline p = Pipeline.create(options);
  PCollection<Long> count = p
      .apply(BigtableIO.read().withBigtableOptions(bigtableOptionsBuilder).withTableId(tableId))
      .apply(Count.<Row>globally());
  PAssert.thatSingleton(count).isEqualTo(numRows);
  p.run();
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@Parameters(name = "{index}: {0}")
public static Iterable<? extends PipelineOptions> options() {
  PipelineOptionsFactory.register(TestUnserializableOptions.class);
  PipelineOptionsFactory.register(TestDefaultOptions.class);
  PipelineOptionsFactory.register(TestOptions.class);
  PipelineOptions emptyOptions = PipelineOptionsFactory.create();

  TestUnserializableOptions withNonSerializable =
      PipelineOptionsFactory.as(TestUnserializableOptions.class);
  withNonSerializable.setUnserializable(new Object());

  TestOptions withCustomField = PipelineOptionsFactory.as(TestOptions.class);
  withCustomField.setExample(99);

  PipelineOptions withSettings = PipelineOptionsFactory.create();
  withSettings.as(ApplicationNameOptions.class).setAppName("my_app");
  withSettings.setJobName("my_job");

  PipelineOptions withParsedSettings =
      PipelineOptionsFactory.fromArgs("--jobName=my_job --appName=my_app").create();

  return ImmutableList.of(
      emptyOptions, withNonSerializable, withCustomField, withSettings, withParsedSettings);
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
public static void main(String[] args) {
    PipelineOptionsFactory.register(TemplateOptions.class);
    TemplateOptions options = PipelineOptionsFactory.fromArgs(args).withValidation().as(TemplateOptions.class);
    Pipeline pipeline = Pipeline.create(options);
    pipeline.apply("READ", TextIO.read().from(options.getInputFile()).withCompressionType(GZIP))
            .apply("TRANSFORM", ParDo.of(new WikiParDo()))
            .apply("WRITE", BigQueryIO.writeTableRows()
                    .to(String.format("%s:devfest_melbourne_2017.wiki_demo", options.getProject()))
                    .withCreateDisposition(CREATE_IF_NEEDED)
                    .withWriteDisposition(WRITE_APPEND)
                    .withSchema(getTableSchema()));
    pipeline.run();
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@Before
public void setup() {
  PipelineOptionsFactory.register(V1TestOptions.class);
  options = TestPipeline.testingPipelineOptions().as(V1TestOptions.class);
  project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject();
  ancestor = UUID.randomUUID().toString();
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@Before
public void setup() throws Exception {
  PipelineOptionsFactory.register(V1TestOptions.class);
  options = TestPipeline.testingPipelineOptions().as(V1TestOptions.class);
  project = TestPipeline.testingPipelineOptions().as(GcpOptions.class).getProject();
  ancestor = UUID.randomUUID().toString();
  // Create entities and write them to datastore
  writeEntitiesToDatastore(options, project, ancestor, numEntities);
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@Before
public void setUp() throws Exception {
  PipelineOptionsFactory.register(SpannerTestPipelineOptions.class);
  options = TestPipeline.testingPipelineOptions().as(SpannerTestPipelineOptions.class);

  project = options.as(GcpOptions.class).getProject();

  spanner = SpannerOptions.newBuilder().setProjectId(project).build().getService();

  databaseName = generateDatabaseName();

  databaseAdminClient = spanner.getDatabaseAdminClient();

  // Delete database if exists.
  databaseAdminClient.dropDatabase(options.getInstanceId(), databaseName);

  Operation<Database, CreateDatabaseMetadata> op =
      databaseAdminClient.createDatabase(
          options.getInstanceId(),
          databaseName,
          Collections.singleton(
              "CREATE TABLE "
                  + options.getTable()
                  + " ("
                  + "  Key           INT64,"
                  + "  Value         STRING(MAX),"
                  + ") PRIMARY KEY (Key)"));
  op.waitFor();
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@BeforeClass
public static void setup() throws SQLException, ParseException {
  PipelineOptionsFactory.register(IOTestPipelineOptions.class);
  IOTestPipelineOptions options = TestPipeline.testingPipelineOptions()
      .as(IOTestPipelineOptions.class);

  dataSource = getDataSource(options);

  tableName = JdbcTestHelper.getTableName("IT");
  JdbcTestHelper.createDataTable(dataSource, tableName);
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
public static IOTestPipelineOptions readTestPipelineOptions() {
  PipelineOptionsFactory.register(IOTestPipelineOptions.class);
  IOTestPipelineOptions options = TestPipeline
      .testingPipelineOptions()
      .as(IOTestPipelineOptions.class);

  return PipelineOptionsValidator.validate(IOTestPipelineOptions.class, options);
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@BeforeClass
public static void beforeClass() throws Exception {
  PipelineOptionsFactory.register(IOTestPipelineOptions.class);
  options = TestPipeline.testingPipelineOptions().as(IOTestPipelineOptions.class);
  readConnectionConfiguration = ElasticsearchIOITCommon
      .getConnectionConfiguration(options, ElasticsearchIOITCommon.ReadOrWrite.READ);
  writeConnectionConfiguration = ElasticsearchIOITCommon
      .getConnectionConfiguration(options, ElasticsearchIOITCommon.ReadOrWrite.WRITE);
  restClient = readConnectionConfiguration.createClient();
  elasticsearchIOTestCommon = new ElasticsearchIOTestCommon(readConnectionConfiguration,
      restClient, true);
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@Test
public void testWordCountExample() throws Exception {
  PipelineOptionsFactory.register(WordCountOptions.class);
  WordCountOptions options = TestPipeline.testingPipelineOptions().as(WordCountOptions.class);
  options.setRunner(TestApexRunner.class);
  options.setApplicationName("StreamingWordCount");
  String inputFile = WordCountTest.class.getResource("/words.txt").getFile();
  options.setInputFile(new File(inputFile).getAbsolutePath());
  String outputFilePrefix = "target/wordcountresult.txt";
  options.setOutput(outputFilePrefix);

  File outFile1 = new File(outputFilePrefix + "-00000-of-00002");
  File outFile2 = new File(outputFilePrefix + "-00001-of-00002");
  Assert.assertTrue(!outFile1.exists() || outFile1.delete());
  Assert.assertTrue(!outFile2.exists() || outFile2.delete());

  WordCountTest.main(TestPipeline.convertToArgs(options));

  Assert.assertTrue("result files exist", outFile1.exists() && outFile2.exists());
  HashSet<String> results = new HashSet<>();
  results.addAll(FileUtils.readLines(outFile1));
  results.addAll(FileUtils.readLines(outFile2));
  HashSet<String> expectedOutput = Sets.newHashSet(
      "foo - 5 @ 294247-01-09T04:00:54.775Z",
      "bar - 5 @ 294247-01-09T04:00:54.775Z"
  );
  Assert.assertEquals("expected output", expectedOutput, results);
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@BeforeClass
public static void setUp() {
  PipelineOptionsFactory.register(TestPipelineOptions.class);
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@BeforeClass
public static void setUp() {
  PipelineOptionsFactory.register(HIFTestOptions.class);
  options = TestPipeline.testingPipelineOptions().as(HIFTestOptions.class);
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@BeforeClass
public static void setup() {
  PipelineOptionsFactory.register(KinesisTestOptions.class);
  options = TestPipeline.testingPipelineOptions().as(KinesisTestOptions.class);
}