Java PipelineOptionsFactory.as方法代码示例

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
/** Deploys the invoicing pipeline as a template on GCS, for a given projectID and GCS bucket. */
public void deploy() {
  // We can't store options as a member variable due to serialization concerns.
  InvoicingPipelineOptions options = PipelineOptionsFactory.as(InvoicingPipelineOptions.class);
  options.setProject(projectId);
  options.setRunner(DataflowRunner.class);
  options.setStagingLocation(beamBucket + "/staging");
  options.setTemplateLocation(beamBucket + "/templates/invoicing");
  Pipeline p = Pipeline.create(options);

  PCollection<BillingEvent> billingEvents =
      p.apply(
          "Read BillingEvents from Bigquery",
          BigQueryIO.read(BillingEvent::parseFromRecord)
              .fromQuery(InvoicingUtils.makeQueryProvider(options.getYearMonth(), projectId))
              .withCoder(SerializableCoder.of(BillingEvent.class))
              .usingStandardSql()
              .withoutValidation()
              .withTemplateCompatibility());
  applyTerminalTransforms(billingEvents, options.getYearMonth());
  p.run();
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
/**
 * Tests that the {@link DataflowRunner} with {@code --templateLocation} throws the appropriate
 * exception when an output file is not writable.
 */
@Test
public void testTemplateRunnerLoggedErrorForFile() throws Exception {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setJobName("TestJobName");
  options.setRunner(DataflowRunner.class);
  options.setTemplateLocation("//bad/path");
  options.setProject("test-project");
  options.setTempLocation(tmpFolder.getRoot().getPath());
  options.setGcpCredential(new TestCredential());
  options.setPathValidatorClass(NoopPathValidator.class);
  Pipeline p = Pipeline.create(options);

  thrown.expectMessage("Cannot create output file at");
  thrown.expect(RuntimeException.class);
  p.run();
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
/**
 * Tests that the {@link DataflowRunner} with {@code --templateLocation} returns normally when the
 * runner is successfully run.
 */
@Test
public void testTemplateRunnerFullCompletion() throws Exception {
  File existingFile = tmpFolder.newFile();
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setJobName("TestJobName");
  options.setGcpCredential(new TestCredential());
  options.setPathValidatorClass(NoopPathValidator.class);
  options.setProject("test-project");
  options.setRunner(DataflowRunner.class);
  options.setTemplateLocation(existingFile.getPath());
  options.setTempLocation(tmpFolder.getRoot().getPath());
  Pipeline p = Pipeline.create(options);

  p.run();
  expectedLogs.verifyInfo("Template successfully created");
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@Parameters(name = "{index}: {0}")
public static Iterable<? extends PipelineOptions> options() {
  PipelineOptionsFactory.register(TestUnserializableOptions.class);
  PipelineOptionsFactory.register(TestDefaultOptions.class);
  PipelineOptionsFactory.register(TestOptions.class);
  PipelineOptions emptyOptions = PipelineOptionsFactory.create();

  TestUnserializableOptions withNonSerializable =
      PipelineOptionsFactory.as(TestUnserializableOptions.class);
  withNonSerializable.setUnserializable(new Object());

  TestOptions withCustomField = PipelineOptionsFactory.as(TestOptions.class);
  withCustomField.setExample(99);

  PipelineOptions withSettings = PipelineOptionsFactory.create();
  withSettings.as(ApplicationNameOptions.class).setAppName("my_app");
  withSettings.setJobName("my_job");

  PipelineOptions withParsedSettings =
      PipelineOptionsFactory.fromArgs("--jobName=my_job --appName=my_app").create();

  return ImmutableList.of(
      emptyOptions, withNonSerializable, withCustomField, withSettings, withParsedSettings);
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@Test
public void testStagingLocation() {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setPathValidatorClass(NoopPathValidator.class);
  options.setTempLocation("gs://temp_location");
  options.setStagingLocation("gs://staging_location");
  assertEquals("gs://temp_location", options.getGcpTempLocation());
  assertEquals("gs://staging_location", options.getStagingLocation());
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@Test
public void testDefaultInvalidGcpTempLocation() {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setGcpTempLocation("file://temp_location");
  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage(
      "Error constructing default value for stagingLocation: gcpTempLocation is not"
      + " a valid GCS path");
  thrown.expectCause(
      hasMessage(containsString("Expected a valid 'gs://' path")));
  options.getStagingLocation();
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@Test
public void testDefaultNoneGcsTempLocation() {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setTempLocation("file://temp_location");
  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage("Error constructing default value for stagingLocation: "
      + "failed to retrieve gcpTempLocation.");
  thrown.expectCause(hasMessage(containsString(
      "Error constructing default value for gcpTempLocation")));
  options.getStagingLocation();
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@Test
public void sourceAndReadersWork() throws Exception {
  NexmarkOptions options = PipelineOptionsFactory.as(NexmarkOptions.class);
  long n = 200L;
  BoundedEventSource source = new BoundedEventSource(makeConfig(n), 1);

  SourceTestUtils.assertUnstartedReaderReadsSameAsItsSource(
      source.createReader(options), options);
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
private static SparkContextOptions getSparkContextOptions(JavaSparkContext jsc) {
    final SparkContextOptions options = PipelineOptionsFactory.as(SparkContextOptions.class);
    options.setRunner(TestSparkRunner.class);
    options.setUsesProvidedSparkContext(true);
    options.setProvidedSparkContext(jsc);
    options.setEnableSparkMetricSinks(false);
    return options;
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@Test
public void testUserNameIsNotSet() {
  resetDateTimeProviderRule.setDateTimeFixed("2014-12-08T19:07:06.698Z");
  System.getProperties().remove("user.name");
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setAppName("TestApplication");
  String[] nameComponents = options.getJobName().split("-");
  assertEquals(4, nameComponents.length);
  assertEquals("testapplication", nameComponents[0]);
  assertEquals("", nameComponents[1]);
  assertEquals("1208190706", nameComponents[2]);
  // Verify the last component is a hex integer (unsigned).
  Long.parseLong(nameComponents[3], 16);
  assertTrue(options.getJobName().length() <= 40);
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@Test
public void testNoProjectFails() {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);

  options.setRunner(DataflowRunner.class);
  // Explicitly set to null to prevent the default instance factory from reading credentials
  // from a user's environment, causing this test to fail.
  options.setProject(null);

  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage("Project id");
  thrown.expectMessage("when running a Dataflow in the cloud");

  DataflowRunner.fromOptions(options);
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@Test
public void customSettingsRetained() throws Exception {
  TestOptions options = PipelineOptionsFactory.as(TestOptions.class);
  options.setExample(23);
  Struct serialized = PipelineOptionsTranslation.toProto(options);
  PipelineOptions deserialized = PipelineOptionsTranslation.fromProto(serialized);

  assertThat(deserialized.as(TestOptions.class).getExample(), equalTo(23));
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@Test
public void testDefaultGcpTempLocationInvalid() throws Exception {
  GcpOptions options = PipelineOptionsFactory.as(GcpOptions.class);
  options.setTempLocation("file://");
  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage(
      "Error constructing default value for gcpTempLocation: tempLocation is not"
          + " a valid GCS path");
  options.getGcpTempLocation();
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@Test
public void testDefaultGcpTempLocationDoesNotExist() {
  GcpOptions options = PipelineOptionsFactory.as(GcpOptions.class);
  String tempLocation = "gs://does/not/exist";
  options.setTempLocation(tempLocation);
  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage(
      "Error constructing default value for gcpTempLocation: tempLocation is not"
          + " a valid GCS path");
  thrown.expectCause(
      hasMessage(containsString("Output path does not exist or is not writeable")));

  options.getGcpTempLocation();
}
 

import org.apache.beam.sdk.options.PipelineOptionsFactory; //导入方法依赖的package包/类
@Test
public void testProjectDescription() throws IOException {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setRunner(DataflowRunner.class);
  options.setProject("some project");

  options.setGcpTempLocation(VALID_TEMP_BUCKET);
  options.setGcsUtil(mockGcsUtil);

  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage("Project ID");
  thrown.expectMessage("project description");

  DataflowRunner.fromOptions(options);
}