Java Job.getSteps方法代码示例

import com.google.api.services.dataflow.model.Job; //导入方法依赖的package包/类
private static void assertAllStepOutputsHaveUniqueIds(Job job)
    throws Exception {
  List<Long> outputIds = new ArrayList<>();
  for (Step step : job.getSteps()) {
    List<Map<String, Object>> outputInfoList =
        (List<Map<String, Object>>) step.getProperties().get(PropertyNames.OUTPUT_INFO);
    if (outputInfoList != null) {
      for (Map<String, Object> outputInfo : outputInfoList) {
        outputIds.add(Long.parseLong(Structs.getString(outputInfo, PropertyNames.OUTPUT_NAME)));
      }
    }
  }
  Set<Long> uniqueOutputNames = new HashSet<>(outputIds);
  outputIds.removeAll(uniqueOutputNames);
  assertTrue(String.format("Found duplicate output ids %s", outputIds),
      outputIds.size() == 0);
}
 

import com.google.api.services.dataflow.model.Job; //导入方法依赖的package包/类
@Override
public boolean matches(Object o) {
  Job job = (Job) o;
  return job.getId() == null
      && job.getProjectId() == null
      && job.getName() != null
      && job.getType() != null
      && job.getEnvironment() != null
      && job.getSteps() != null
      && job.getCurrentState() == null
      && job.getCurrentStateTime() == null
      && job.getExecutionInfo() == null
      && job.getCreateTime() == null;
}
 

import com.google.api.services.dataflow.model.Job; //导入方法依赖的package包/类
@Test
public void testToSingletonTranslationWithIsmSideInput() throws Exception {
  // A "change detector" test that makes sure the translation
  // of getting a PCollectionView<T> does not change
  // in bad ways during refactor

  DataflowPipelineOptions options = buildPipelineOptions();
  DataflowPipelineTranslator translator = DataflowPipelineTranslator.fromOptions(options);

  Pipeline pipeline = Pipeline.create(options);
  pipeline.apply(Create.of(1))
      .apply(View.<Integer>asSingleton());
  DataflowRunner runner = DataflowRunner.fromOptions(options);
  runner.replaceTransforms(pipeline);
  Job job =
      translator
          .translate(
              pipeline,
              runner,
              Collections.<DataflowPackage>emptyList())
          .getJob();
  assertAllStepOutputsHaveUniqueIds(job);

  List<Step> steps = job.getSteps();
  assertEquals(9, steps.size());

  @SuppressWarnings("unchecked")
  List<Map<String, Object>> toIsmRecordOutputs =
      (List<Map<String, Object>>) steps.get(7).getProperties().get(PropertyNames.OUTPUT_INFO);
  assertTrue(
      Structs.getBoolean(Iterables.getOnlyElement(toIsmRecordOutputs), "use_indexed_format"));

  Step collectionToSingletonStep = steps.get(8);
  assertEquals("CollectionToSingleton", collectionToSingletonStep.getKind());
}
 

import com.google.api.services.dataflow.model.Job; //导入方法依赖的package包/类
@Test
public void testToIterableTranslationWithIsmSideInput() throws Exception {
  // A "change detector" test that makes sure the translation
  // of getting a PCollectionView<Iterable<T>> does not change
  // in bad ways during refactor

  DataflowPipelineOptions options = buildPipelineOptions();
  DataflowPipelineTranslator translator = DataflowPipelineTranslator.fromOptions(options);

  Pipeline pipeline = Pipeline.create(options);
  pipeline.apply(Create.of(1, 2, 3))
      .apply(View.<Integer>asIterable());

  DataflowRunner runner = DataflowRunner.fromOptions(options);
  runner.replaceTransforms(pipeline);
  Job job =
      translator.translate(pipeline, runner, Collections.<DataflowPackage>emptyList()).getJob();
  assertAllStepOutputsHaveUniqueIds(job);

  List<Step> steps = job.getSteps();
  assertEquals(3, steps.size());

  @SuppressWarnings("unchecked")
  List<Map<String, Object>> toIsmRecordOutputs =
      (List<Map<String, Object>>) steps.get(1).getProperties().get(PropertyNames.OUTPUT_INFO);
  assertTrue(
      Structs.getBoolean(Iterables.getOnlyElement(toIsmRecordOutputs), "use_indexed_format"));


  Step collectionToSingletonStep = steps.get(2);
  assertEquals("CollectionToSingleton", collectionToSingletonStep.getKind());
}
 

import com.google.api.services.dataflow.model.Job; //导入方法依赖的package包/类
/**
 * Smoke test to fail fast if translation of a stateful ParDo
 * in batch breaks.
 */
@Test
public void testBatchStatefulParDoTranslation() throws Exception {
  DataflowPipelineOptions options = buildPipelineOptions();
  DataflowRunner runner = DataflowRunner.fromOptions(options);
  options.setStreaming(false);
  DataflowPipelineTranslator translator = DataflowPipelineTranslator.fromOptions(options);

  Pipeline pipeline = Pipeline.create(options);

  TupleTag<Integer> mainOutputTag = new TupleTag<Integer>() {};

  pipeline
      .apply(Create.of(KV.of(1, 1)))
      .apply(
          ParDo.of(
              new DoFn<KV<Integer, Integer>, Integer>() {
                @StateId("unused")
                final StateSpec<ValueState<Integer>> stateSpec =
                    StateSpecs.value(VarIntCoder.of());

                @ProcessElement
                public void process(ProcessContext c) {
                  // noop
                }
              }).withOutputTags(mainOutputTag, TupleTagList.empty()));

  runner.replaceTransforms(pipeline);

  Job job =
      translator
          .translate(
              pipeline,
              runner,
              Collections.<DataflowPackage>emptyList())
          .getJob();

  // The job should look like:
  // 0. ParallelRead (Create)
  // 1. ParDo(ReifyWVs)
  // 2. GroupByKeyAndSortValuesONly
  // 3. A ParDo over grouped and sorted KVs that is executed via ungrouping service-side

  List<Step> steps = job.getSteps();
  assertEquals(4, steps.size());

  Step createStep = steps.get(0);
  assertEquals("ParallelRead", createStep.getKind());

  Step reifyWindowedValueStep = steps.get(1);
  assertEquals("ParallelDo", reifyWindowedValueStep.getKind());

  Step gbkStep = steps.get(2);
  assertEquals("GroupByKey", gbkStep.getKind());

  Step statefulParDoStep = steps.get(3);
  assertEquals("ParallelDo", statefulParDoStep.getKind());
  assertThat(
      (String) statefulParDoStep.getProperties().get(PropertyNames.USES_KEYED_STATE),
      not(equalTo("true")));
}
 

import com.google.api.services.dataflow.model.Job; //导入方法依赖的package包/类
/**
 * Smoke test to fail fast if translation of a splittable ParDo
 * in streaming breaks.
 */
@Test
public void testStreamingSplittableParDoTranslation() throws Exception {
  DataflowPipelineOptions options = buildPipelineOptions();
  DataflowRunner runner = DataflowRunner.fromOptions(options);
  options.setStreaming(true);
  DataflowPipelineTranslator translator = DataflowPipelineTranslator.fromOptions(options);

  Pipeline pipeline = Pipeline.create(options);

  PCollection<String> windowedInput = pipeline
      .apply(Create.of("a"))
      .apply(Window.<String>into(FixedWindows.of(Duration.standardMinutes(1))));
  windowedInput.apply(ParDo.of(new TestSplittableFn()));

  runner.replaceTransforms(pipeline);

  Job job =
      translator
          .translate(
              pipeline,
              runner,
              Collections.<DataflowPackage>emptyList())
          .getJob();

  // The job should contain a SplittableParDo.ProcessKeyedElements step, translated as
  // "SplittableProcessKeyed".

  List<Step> steps = job.getSteps();
  Step processKeyedStep = null;
  for (Step step : steps) {
    if (step.getKind().equals("SplittableProcessKeyed")) {
      assertNull(processKeyedStep);
      processKeyedStep = step;
    }
  }
  assertNotNull(processKeyedStep);

  @SuppressWarnings({"unchecked", "rawtypes"})
  DoFnInfo<String, Integer> fnInfo =
      (DoFnInfo<String, Integer>)
          SerializableUtils.deserializeFromByteArray(
              jsonStringToByteArray(
                  Structs.getString(
                      processKeyedStep.getProperties(), PropertyNames.SERIALIZED_FN)),
              "DoFnInfo");
  assertThat(fnInfo.getDoFn(), instanceOf(TestSplittableFn.class));
  assertThat(
      fnInfo.getWindowingStrategy().getWindowFn(),
      Matchers.<WindowFn>equalTo(FixedWindows.of(Duration.standardMinutes(1))));
  Coder<?> restrictionCoder =
      CloudObjects.coderFromCloudObject(
          (CloudObject)
              Structs.getObject(
                  processKeyedStep.getProperties(), PropertyNames.RESTRICTION_CODER));

  assertEquals(SerializableCoder.of(OffsetRange.class), restrictionCoder);
}