Java DataflowPipelineOptions.setProject方法代码示例

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
/**
 * Tests that the {@link DataflowRunner} with {@code --templateLocation} returns normally when the
 * runner is successfully run.
 */
@Test
public void testTemplateRunnerFullCompletion() throws Exception {
  File existingFile = tmpFolder.newFile();
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setJobName("TestJobName");
  options.setGcpCredential(new TestCredential());
  options.setPathValidatorClass(NoopPathValidator.class);
  options.setProject("test-project");
  options.setRunner(DataflowRunner.class);
  options.setTemplateLocation(existingFile.getPath());
  options.setTempLocation(tmpFolder.getRoot().getPath());
  Pipeline p = Pipeline.create(options);

  p.run();
  expectedLogs.verifyInfo("Template successfully created");
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
private static DataflowPipelineOptions buildPipelineOptions() throws IOException {
  GcsUtil mockGcsUtil = mock(GcsUtil.class);
  when(mockGcsUtil.expand(any(GcsPath.class))).then(new Answer<List<GcsPath>>() {
    @Override
    public List<GcsPath> answer(InvocationOnMock invocation) throws Throwable {
      return ImmutableList.of((GcsPath) invocation.getArguments()[0]);
    }
  });
  when(mockGcsUtil.bucketAccessible(any(GcsPath.class))).thenReturn(true);

  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setRunner(DataflowRunner.class);
  options.setGcpCredential(new TestCredential());
  options.setJobName("some-job-name");
  options.setProject("some-project");
  options.setRegion("some-region");
  options.setTempLocation(GcsPath.fromComponents("somebucket", "some/path").toString());
  options.setFilesToStage(new LinkedList<String>());
  options.setDataflowClient(buildMockDataflow(new IsValidCreateRequest()));
  options.setGcsUtil(mockGcsUtil);
  return options;
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
private static DataflowPipelineOptions buildPipelineOptions(String ... args) throws IOException {
  GcsUtil mockGcsUtil = mock(GcsUtil.class);
  when(mockGcsUtil.expand(any(GcsPath.class))).then(new Answer<List<GcsPath>>() {
    @Override
    public List<GcsPath> answer(InvocationOnMock invocation) throws Throwable {
      return ImmutableList.of((GcsPath) invocation.getArguments()[0]);
    }
  });
  when(mockGcsUtil.bucketAccessible(any(GcsPath.class))).thenReturn(true);

  DataflowPipelineOptions options =
      PipelineOptionsFactory.fromArgs(args).as(DataflowPipelineOptions.class);
  options.setRunner(DataflowRunner.class);
  options.setGcpCredential(new TestCredential());
  options.setJobName("some-job-name");
  options.setProject("some-project");
  options.setRegion("some-region");
  options.setTempLocation(GcsPath.fromComponents("somebucket", "some/path").toString());
  options.setFilesToStage(new LinkedList<String>());
  options.setGcsUtil(mockGcsUtil);
  return options;
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
private DataflowPipelineOptions buildPipelineOptions() throws IOException {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setRunner(DataflowRunner.class);
  options.setProject(PROJECT_ID);
  options.setTempLocation(VALID_TEMP_BUCKET);
  options.setRegion(REGION_ID);
  // Set FILES_PROPERTY to empty to prevent a default value calculated from classpath.
  options.setFilesToStage(new LinkedList<String>());
  options.setDataflowClient(buildMockDataflow());
  options.setGcsUtil(mockGcsUtil);
  options.setGcpCredential(new TestCredential());

  // Configure the FileSystem registrar to use these options.
  FileSystems.setDefaultPipelineOptions(options);

  return options;
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testInvalidNumberOfWorkerHarnessThreads() throws IOException {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  FileSystems.setDefaultPipelineOptions(options);
  options.setRunner(DataflowRunner.class);
  options.setProject("foo-12345");

  options.setGcpTempLocation(VALID_TEMP_BUCKET);
  options.setGcsUtil(mockGcsUtil);

  options.as(DataflowPipelineDebugOptions.class).setNumberOfWorkerHarnessThreads(-1);

  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage("Number of worker harness threads");
  thrown.expectMessage("Please make sure the value is non-negative.");

  DataflowRunner.fromOptions(options);
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
/**
 * Create a test pipeline that uses the {@link DataflowRunner} so that {@link GroupByKey}
 * is not expanded. This is used for verifying that even without expansion the proper errors show
 * up.
 */
private Pipeline createTestServiceRunner() {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setRunner(DataflowRunner.class);
  options.setProject("someproject");
  options.setGcpTempLocation("gs://staging");
  options.setPathValidatorClass(NoopPathValidator.class);
  options.setDataflowClient(dataflow);
  return Pipeline.create(options);
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
private Pipeline createTestStreamingRunner() {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setRunner(DataflowRunner.class);
  options.setStreaming(true);
  options.setProject("someproject");
  options.setGcpTempLocation("gs://staging");
  options.setPathValidatorClass(NoopPathValidator.class);
  options.setDataflowClient(dataflow);
  return Pipeline.create(options);
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testDontOverrideEndpointWithDefaultApi() {
  DataflowPipelineOptions options =
      PipelineOptionsFactory.create().as(DataflowPipelineOptions.class);
  options.setProject(PROJECT_ID);
  options.setGcpCredential(new TestCredential());
  String cancelCommand = MonitoringUtil.getGcloudCancelCommand(options, JOB_ID);
  assertEquals("gcloud dataflow jobs --project=someProject cancel 1234", cancelCommand);
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testOverridesEndpointWithStagedDataflowEndpoint() {
  DataflowPipelineOptions options =
      PipelineOptionsFactory.create().as(DataflowPipelineOptions.class);
  options.setProject(PROJECT_ID);
  options.setGcpCredential(new TestCredential());
  String stagingDataflowEndpoint = "v0neverExisted";
  options.setDataflowEndpoint(stagingDataflowEndpoint);
  String cancelCommand = MonitoringUtil.getGcloudCancelCommand(options, JOB_ID);
  assertEquals(
      "CLOUDSDK_API_ENDPOINT_OVERRIDES_DATAFLOW=https://dataflow.googleapis.com/v0neverExisted/ "
      + "gcloud dataflow jobs --project=someProject cancel 1234",
      cancelCommand);
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testGcsStagingLocationInitialization() throws Exception {
  // Set temp location (required), and check that staging location is set.
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setTempLocation(VALID_TEMP_BUCKET);
  options.setProject(PROJECT_ID);
  options.setGcpCredential(new TestCredential());
  options.setGcsUtil(mockGcsUtil);
  options.setRunner(DataflowRunner.class);

  DataflowRunner.fromOptions(options);

  assertNotNull(options.getStagingLocation());
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testNoProjectFails() {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);

  options.setRunner(DataflowRunner.class);
  // Explicitly set to null to prevent the default instance factory from reading credentials
  // from a user's environment, causing this test to fail.
  options.setProject(null);

  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage("Project id");
  thrown.expectMessage("when running a Dataflow in the cloud");

  DataflowRunner.fromOptions(options);
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testProjectId() throws IOException {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setRunner(DataflowRunner.class);
  options.setProject("foo-12345");

  options.setGcpTempLocation(VALID_TEMP_BUCKET);
  options.setGcsUtil(mockGcsUtil);
  options.setGcpCredential(new TestCredential());

  DataflowRunner.fromOptions(options);
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testProjectPrefix() throws IOException {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setRunner(DataflowRunner.class);
  options.setProject("google.com:some-project-12345");

  options.setGcpTempLocation(VALID_TEMP_BUCKET);
  options.setGcsUtil(mockGcsUtil);
  options.setGcpCredential(new TestCredential());

  DataflowRunner.fromOptions(options);
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testProjectNumber() throws IOException {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setRunner(DataflowRunner.class);
  options.setProject("12345");

  options.setGcpTempLocation(VALID_TEMP_BUCKET);
  options.setGcsUtil(mockGcsUtil);

  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage("Project ID");
  thrown.expectMessage("project number");

  DataflowRunner.fromOptions(options);
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testProjectDescription() throws IOException {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setRunner(DataflowRunner.class);
  options.setProject("some project");

  options.setGcpTempLocation(VALID_TEMP_BUCKET);
  options.setGcsUtil(mockGcsUtil);

  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage("Project ID");
  thrown.expectMessage("project description");

  DataflowRunner.fromOptions(options);
}