Java DataflowPipelineOptions.setJobName方法代码示例

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
private static DataflowPipelineOptions buildPipelineOptions() throws IOException {
  GcsUtil mockGcsUtil = mock(GcsUtil.class);
  when(mockGcsUtil.expand(any(GcsPath.class))).then(new Answer<List<GcsPath>>() {
    @Override
    public List<GcsPath> answer(InvocationOnMock invocation) throws Throwable {
      return ImmutableList.of((GcsPath) invocation.getArguments()[0]);
    }
  });
  when(mockGcsUtil.bucketAccessible(any(GcsPath.class))).thenReturn(true);

  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setRunner(DataflowRunner.class);
  options.setGcpCredential(new TestCredential());
  options.setJobName("some-job-name");
  options.setProject("some-project");
  options.setRegion("some-region");
  options.setTempLocation(GcsPath.fromComponents("somebucket", "some/path").toString());
  options.setFilesToStage(new LinkedList<String>());
  options.setDataflowClient(buildMockDataflow(new IsValidCreateRequest()));
  options.setGcsUtil(mockGcsUtil);
  return options;
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
private static DataflowPipelineOptions buildPipelineOptions(String ... args) throws IOException {
  GcsUtil mockGcsUtil = mock(GcsUtil.class);
  when(mockGcsUtil.expand(any(GcsPath.class))).then(new Answer<List<GcsPath>>() {
    @Override
    public List<GcsPath> answer(InvocationOnMock invocation) throws Throwable {
      return ImmutableList.of((GcsPath) invocation.getArguments()[0]);
    }
  });
  when(mockGcsUtil.bucketAccessible(any(GcsPath.class))).thenReturn(true);

  DataflowPipelineOptions options =
      PipelineOptionsFactory.fromArgs(args).as(DataflowPipelineOptions.class);
  options.setRunner(DataflowRunner.class);
  options.setGcpCredential(new TestCredential());
  options.setJobName("some-job-name");
  options.setProject("some-project");
  options.setRegion("some-region");
  options.setTempLocation(GcsPath.fromComponents("somebucket", "some/path").toString());
  options.setFilesToStage(new LinkedList<String>());
  options.setGcsUtil(mockGcsUtil);
  return options;
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testInvalidJobName() throws IOException {
  List<String> invalidNames = Arrays.asList(
      "invalid_name",
      "0invalid",
      "invalid-");
  List<String> expectedReason = Arrays.asList(
      "JobName invalid",
      "JobName invalid",
      "JobName invalid");

  for (int i = 0; i < invalidNames.size(); ++i) {
    DataflowPipelineOptions options = buildPipelineOptions();
    options.setJobName(invalidNames.get(i));

    try {
      DataflowRunner.fromOptions(options);
      fail("Expected IllegalArgumentException for jobName "
          + options.getJobName());
    } catch (IllegalArgumentException e) {
      assertThat(e.getMessage(),
          containsString(expectedReason.get(i)));
    }
  }
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
/**
 * Tests that the {@link DataflowRunner} with {@code --templateLocation} returns normally when the
 * runner is successfully run.
 */
@Test
public void testTemplateRunnerFullCompletion() throws Exception {
  File existingFile = tmpFolder.newFile();
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setJobName("TestJobName");
  options.setGcpCredential(new TestCredential());
  options.setPathValidatorClass(NoopPathValidator.class);
  options.setProject("test-project");
  options.setRunner(DataflowRunner.class);
  options.setTemplateLocation(existingFile.getPath());
  options.setTempLocation(tmpFolder.getRoot().getPath());
  Pipeline p = Pipeline.create(options);

  p.run();
  expectedLogs.verifyInfo("Template successfully created");
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
/**
 * Tests that the {@link DataflowRunner} with {@code --templateLocation} throws the appropriate
 * exception when an output file is not writable.
 */
@Test
public void testTemplateRunnerLoggedErrorForFile() throws Exception {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setJobName("TestJobName");
  options.setRunner(DataflowRunner.class);
  options.setTemplateLocation("//bad/path");
  options.setProject("test-project");
  options.setTempLocation(tmpFolder.getRoot().getPath());
  options.setGcpCredential(new TestCredential());
  options.setPathValidatorClass(NoopPathValidator.class);
  Pipeline p = Pipeline.create(options);

  thrown.expectMessage("Cannot create output file at");
  thrown.expect(RuntimeException.class);
  p.run();
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testFromOptionsWithUppercaseConvertsToLowercase() throws Exception {
  String mixedCase = "ThisJobNameHasMixedCase";
  DataflowPipelineOptions options = buildPipelineOptions();
  options.setJobName(mixedCase);

  DataflowRunner.fromOptions(options);
  assertThat(options.getJobName(), equalTo(mixedCase.toLowerCase()));
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
/**
 * Invasive mock-based test for checking that the JSON generated for the pipeline options has
 * not had vital fields pruned.
 */
@Test
public void testSettingOfSdkPipelineOptions() throws IOException {
  DataflowPipelineOptions options = buildPipelineOptions();

  // These options are important only for this test, and need not be global to the test class
  options.setAppName(DataflowRunnerTest.class.getSimpleName());
  options.setJobName("some-job-name");

  Pipeline p = Pipeline.create(options);
  p.run();

  ArgumentCaptor<Job> jobCaptor = ArgumentCaptor.forClass(Job.class);
  Mockito.verify(mockJobs).create(eq(PROJECT_ID), eq(REGION_ID), jobCaptor.capture());

  Map<String, Object> sdkPipelineOptions =
      jobCaptor.getValue().getEnvironment().getSdkPipelineOptions();

  assertThat(sdkPipelineOptions, hasKey("options"));
  Map<String, Object> optionsMap = (Map<String, Object>) sdkPipelineOptions.get("options");

  assertThat(optionsMap, hasEntry("appName", (Object) options.getAppName()));
  assertThat(optionsMap, hasEntry("project", (Object) options.getProject()));
  assertThat(
      optionsMap,
      hasEntry("pathValidatorClass", (Object) options.getPathValidatorClass().getName()));
  assertThat(optionsMap, hasEntry("runner", (Object) options.getRunner().getName()));
  assertThat(optionsMap, hasEntry("jobName", (Object) options.getJobName()));
  assertThat(optionsMap, hasEntry("tempLocation", (Object) options.getTempLocation()));
  assertThat(
      optionsMap, hasEntry("stagingLocation", (Object) options.getStagingLocation()));
  assertThat(
      optionsMap,
      hasEntry("stableUniqueNames", (Object) options.getStableUniqueNames().toString()));
  assertThat(optionsMap, hasEntry("streaming", (Object) options.isStreaming()));
  assertThat(
      optionsMap,
      hasEntry(
          "numberOfWorkerHarnessThreads", (Object) options.getNumberOfWorkerHarnessThreads()));
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testUpdate() throws IOException {
  DataflowPipelineOptions options = buildPipelineOptions();
  options.setUpdate(true);
  options.setJobName("oldJobName");
  Pipeline p = buildDataflowPipeline(options);
  DataflowPipelineJob job = (DataflowPipelineJob) p.run();
  assertEquals("newid", job.getJobId());

  ArgumentCaptor<Job> jobCaptor = ArgumentCaptor.forClass(Job.class);
  Mockito.verify(mockJobs).create(eq(PROJECT_ID), eq(REGION_ID), jobCaptor.capture());
  assertValidJob(jobCaptor.getValue());
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testUpdateNonExistentPipeline() throws IOException {
  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage("Could not find running job named badjobname");

  DataflowPipelineOptions options = buildPipelineOptions();
  options.setUpdate(true);
  options.setJobName("badJobName");
  Pipeline p = buildDataflowPipeline(options);
  p.run();
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testUpdateAlreadyUpdatedPipeline() throws IOException {
  DataflowPipelineOptions options = buildPipelineOptions();
  options.setUpdate(true);
  options.setJobName("oldJobName");
  Dataflow mockDataflowClient = options.getDataflowClient();
  Dataflow.Projects.Locations.Jobs.Create mockRequest = mock(
      Dataflow.Projects.Locations.Jobs.Create.class);
  when(mockDataflowClient.projects().locations().jobs()
      .create(eq(PROJECT_ID), eq(REGION_ID), any(Job.class)))
      .thenReturn(mockRequest);
  final Job resultJob = new Job();
  resultJob.setId("newid");
  // Return a different request id.
  resultJob.setClientRequestId("different_request_id");
  when(mockRequest.execute()).thenReturn(resultJob);

  Pipeline p = buildDataflowPipeline(options);

  thrown.expect(DataflowJobAlreadyUpdatedException.class);
  thrown.expect(new TypeSafeMatcher<DataflowJobAlreadyUpdatedException>() {
    @Override
    public void describeTo(Description description) {
      description.appendText("Expected job ID: " + resultJob.getId());
    }

    @Override
    protected boolean matchesSafely(DataflowJobAlreadyUpdatedException item) {
      return resultJob.getId().equals(item.getJob().getJobId());
    }
  });
  thrown.expectMessage("The job named oldjobname with id: oldJobId has already been updated "
      + "into job id: newid and cannot be updated again.");
  p.run();
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testValidJobName() throws IOException {
  List<String> names = Arrays.asList("ok", "Ok", "A-Ok", "ok-123",
      "this-one-is-fairly-long-01234567890123456789");

  for (String name : names) {
    DataflowPipelineOptions options = buildPipelineOptions();
    options.setJobName(name);

    DataflowRunner runner = DataflowRunner
        .fromOptions(options);
    assertNotNull(runner);
  }
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testToString() {
  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setJobName("TestJobName");
  options.setProject("test-project");
  options.setTempLocation("gs://test/temp/location");
  options.setGcpCredential(new TestCredential());
  options.setPathValidatorClass(NoopPathValidator.class);
  options.setRunner(DataflowRunner.class);
  assertEquals(
      "DataflowRunner#testjobname",
      DataflowRunner.fromOptions(options).toString());
}
 

import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; //导入方法依赖的package包/类
@Test
public void testRunWithFiles() throws IOException {
  // Test that the function DataflowRunner.stageFiles works as expected.
  final String cloudDataflowDataset = "somedataset";

  // Create some temporary files.
  File temp1 = File.createTempFile("DataflowRunnerTest", "txt");
  temp1.deleteOnExit();
  File temp2 = File.createTempFile("DataflowRunnerTest2", "txt");
  temp2.deleteOnExit();

  String overridePackageName = "alias.txt";

  when(mockGcsUtil.getObjects(anyListOf(GcsPath.class)))
      .thenReturn(ImmutableList.of(GcsUtil.StorageObjectOrIOException.create(
          new FileNotFoundException("some/path"))));

  DataflowPipelineOptions options = PipelineOptionsFactory.as(DataflowPipelineOptions.class);
  options.setFilesToStage(ImmutableList.of(
      temp1.getAbsolutePath(),
      overridePackageName + "=" + temp2.getAbsolutePath()));
  options.setStagingLocation(VALID_STAGING_BUCKET);
  options.setTempLocation(VALID_TEMP_BUCKET);
  options.setTempDatasetId(cloudDataflowDataset);
  options.setProject(PROJECT_ID);
  options.setRegion(REGION_ID);
  options.setJobName("job");
  options.setDataflowClient(buildMockDataflow());
  options.setGcsUtil(mockGcsUtil);
  options.setGcpCredential(new TestCredential());

  when(mockGcsUtil.create(any(GcsPath.class), anyString(), anyInt()))
      .then(
          new Answer<SeekableByteChannel>() {
            @Override
            public SeekableByteChannel answer(InvocationOnMock invocation) throws Throwable {
              return FileChannel.open(
                  Files.createTempFile("channel-", ".tmp"),
                  StandardOpenOption.CREATE,
                  StandardOpenOption.WRITE,
                  StandardOpenOption.DELETE_ON_CLOSE);
            }
          });

  Pipeline p = buildDataflowPipeline(options);

  DataflowPipelineJob job = (DataflowPipelineJob) p.run();
  assertEquals("newid", job.getJobId());

  ArgumentCaptor<Job> jobCaptor = ArgumentCaptor.forClass(Job.class);
  Mockito.verify(mockJobs).create(eq(PROJECT_ID), eq(REGION_ID), jobCaptor.capture());
  Job workflowJob = jobCaptor.getValue();
  assertValidJob(workflowJob);

  assertEquals(
      2,
      workflowJob.getEnvironment().getWorkerPools().get(0).getPackages().size());
  DataflowPackage workflowPackage1 =
      workflowJob.getEnvironment().getWorkerPools().get(0).getPackages().get(0);
  assertThat(workflowPackage1.getName(), startsWith(temp1.getName()));
  DataflowPackage workflowPackage2 =
      workflowJob.getEnvironment().getWorkerPools().get(0).getPackages().get(1);
  assertEquals(overridePackageName, workflowPackage2.getName());

  assertEquals(
      GcsPath.fromUri(VALID_TEMP_BUCKET).toResourceName(),
      workflowJob.getEnvironment().getTempStoragePrefix());
  assertEquals(
      cloudDataflowDataset,
      workflowJob.getEnvironment().getDataset());
  assertEquals(
      DataflowRunnerInfo.getDataflowRunnerInfo().getName(),
      workflowJob.getEnvironment().getUserAgent().get("name"));
  assertEquals(
      DataflowRunnerInfo.getDataflowRunnerInfo().getVersion(),
      workflowJob.getEnvironment().getUserAgent().get("version"));
}