Java Pipeline.traverseTopologically方法代码示例

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
public static void ensureAllReadsConsumed(Pipeline pipeline) {
  final Set<PCollection<?>> unconsumed = new HashSet<>();
  pipeline.traverseTopologically(
      new PipelineVisitor.Defaults() {
        @Override
        public void visitPrimitiveTransform(Node node) {
          unconsumed.removeAll(node.getInputs().values());
        }

        @Override
        public void visitValue(PValue value, Node producer) {
          if (producer.getTransform() instanceof Read.Bounded
              || producer.getTransform() instanceof Read.Unbounded) {
            unconsumed.add((PCollection<?>) value);
          }
        }
      });
  int i = 0;
  for (PCollection<?> unconsumedPCollection : unconsumed) {
    consume(unconsumedPCollection, i);
    i++;
  }
}
 

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
@Test
public void testTrackSingle() {
  options.setRunner(SparkRunner.class);
  JavaSparkContext jsc = SparkContextFactory.getSparkContext(options);
  JavaStreamingContext jssc = new JavaStreamingContext(jsc,
      new org.apache.spark.streaming.Duration(options.getBatchIntervalMillis()));

  Pipeline p = Pipeline.create(options);

  CreateStream<Integer> emptyStream =
      CreateStream.of(
          VarIntCoder.of(),
          Duration.millis(options.getBatchIntervalMillis())).emptyBatch();

  p.apply(emptyStream).apply(ParDo.of(new PassthroughFn<>()));

  p.traverseTopologically(new StreamingSourceTracker(jssc, p, ParDo.MultiOutput.class,  0));
  assertThat(StreamingSourceTracker.numAssertions, equalTo(1));
}
 

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
@Test
public void testWorkerMachineTypeConfig() throws IOException {
  final String testMachineType = "test-machine-type";

  DataflowPipelineOptions options = buildPipelineOptions();
  options.setWorkerMachineType(testMachineType);

  Pipeline p = buildPipeline(options);
  p.traverseTopologically(new RecordingPipelineVisitor());
  Job job =
      DataflowPipelineTranslator.fromOptions(options)
          .translate(
              p, DataflowRunner.fromOptions(options), Collections.<DataflowPackage>emptyList())
          .getJob();

  assertEquals(1, job.getEnvironment().getWorkerPools().size());

  WorkerPool workerPool = job.getEnvironment().getWorkerPools().get(0);
  assertEquals(testMachineType, workerPool.getMachineType());
}
 

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
@Test
public void testApplyIsScopedToExactClass() throws IOException {
  DataflowPipelineOptions options = buildPipelineOptions();
  Pipeline p = Pipeline.create(options);

  Create.TimestampedValues<String> transform =
      Create.timestamped(Arrays.asList(TimestampedValue.of("TestString", Instant.now())));
  p.apply(transform);

  CompositeTransformRecorder recorder = new CompositeTransformRecorder();
  p.traverseTopologically(recorder);

  // The recorder will also have seen a Create.Values composite as well, but we can't obtain that
  // transform.
  assertThat(
      "Expected to have seen CreateTimestamped composite transform.",
      recorder.getCompositeTransforms(),
      hasItem(transform));
  assertThat(
      "Expected to have two composites, CreateTimestamped and Create.Values",
      recorder.getCompositeTransforms(),
      hasItem(Matchers.<PTransform<?, ?>>isA((Class) Create.Values.class)));
}
 

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
@Test
public void testDiskSizeGbConfig() throws IOException {
  final Integer diskSizeGb = 1234;

  DataflowPipelineOptions options = buildPipelineOptions();
  options.setDiskSizeGb(diskSizeGb);

  Pipeline p = buildPipeline(options);
  p.traverseTopologically(new RecordingPipelineVisitor());
  Job job =
      DataflowPipelineTranslator.fromOptions(options)
          .translate(
              p, DataflowRunner.fromOptions(options), Collections.<DataflowPackage>emptyList())
          .getJob();

  assertEquals(1, job.getEnvironment().getWorkerPools().size());
  assertEquals(diskSizeGb,
      job.getEnvironment().getWorkerPools().get(0).getDiskSizeGb());
}
 

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
@Test
public void testSubnetworkConfig() throws IOException {
  final String testSubnetwork = "regions/REGION/subnetworks/SUBNETWORK";

  DataflowPipelineOptions options = buildPipelineOptions();
  options.setSubnetwork(testSubnetwork);

  Pipeline p = buildPipeline(options);
  p.traverseTopologically(new RecordingPipelineVisitor());
  Job job =
      DataflowPipelineTranslator.fromOptions(options)
          .translate(
              p, DataflowRunner.fromOptions(options), Collections.<DataflowPackage>emptyList())
          .getJob();

  assertEquals(1, job.getEnvironment().getWorkerPools().size());
  assertEquals(testSubnetwork,
      job.getEnvironment().getWorkerPools().get(0).getSubnetwork());
}
 

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
public boolean isStreaming(Pipeline pipeline) {
  if(options.isStreaming()) {
    return true;
  }

  pipeline.traverseTopologically(this);

  return streaming;
}
 

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
@Test
public void testSubnetworkConfigMissing() throws IOException {
  DataflowPipelineOptions options = buildPipelineOptions();

  Pipeline p = buildPipeline(options);
  p.traverseTopologically(new RecordingPipelineVisitor());
  Job job =
      DataflowPipelineTranslator.fromOptions(options)
          .translate(
              p, DataflowRunner.fromOptions(options), Collections.<DataflowPackage>emptyList())
          .getJob();

  assertEquals(1, job.getEnvironment().getWorkerPools().size());
  assertNull(job.getEnvironment().getWorkerPools().get(0).getSubnetwork());
}
 

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
@Test
public void testProtoAgainstRehydrated() throws Exception {
  RunnerApi.Pipeline pipelineProto = PipelineTranslation.toProto(pipeline);
  Pipeline rehydrated = PipelineTranslation.fromProto(pipelineProto);

  rehydrated.traverseTopologically(
      new PipelineProtoVerificationVisitor(pipelineProto));
}
 

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
@Test
public void testScalingAlgorithmMissing() throws IOException {
  DataflowPipelineOptions options = buildPipelineOptions();

  Pipeline p = buildPipeline(options);
  p.traverseTopologically(new RecordingPipelineVisitor());
  Job job =
      DataflowPipelineTranslator.fromOptions(options)
          .translate(
              p, DataflowRunner.fromOptions(options), Collections.<DataflowPackage>emptyList())
          .getJob();

  assertEquals(1, job.getEnvironment().getWorkerPools().size());
  // Autoscaling settings are always set.
  assertNull(
      job
          .getEnvironment()
          .getWorkerPools()
          .get(0)
          .getAutoscalingSettings()
          .getAlgorithm());
  assertEquals(
      0,
      job
          .getEnvironment()
          .getWorkerPools()
          .get(0)
          .getAutoscalingSettings()
          .getMaxNumWorkers()
          .intValue());
}
 

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
public void translate(Pipeline pipeline) {
  List<PTransformOverride> overrides =
      ImmutableList.<PTransformOverride>builder()
          .add(PTransformOverride.of(
              PTransformMatchers.classEqualTo(View.CreatePCollectionView.class),
              new CreateStreamingGearpumpView.Factory()))
          .build();

  pipeline.replaceAll(overrides);
  pipeline.traverseTopologically(this);
}
 

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
@Test
public void testMaxNumWorkersIsPassedWhenNoAlgorithmIsSet() throws IOException {
  final DataflowPipelineWorkerPoolOptions.AutoscalingAlgorithmType noScaling = null;
  DataflowPipelineOptions options = buildPipelineOptions();
  options.setMaxNumWorkers(42);
  options.setAutoscalingAlgorithm(noScaling);

  Pipeline p = buildPipeline(options);
  p.traverseTopologically(new RecordingPipelineVisitor());
  Job job =
      DataflowPipelineTranslator.fromOptions(options)
          .translate(
              p, DataflowRunner.fromOptions(options), Collections.<DataflowPackage>emptyList())
          .getJob();

  assertEquals(1, job.getEnvironment().getWorkerPools().size());
  assertNull(
      job
          .getEnvironment()
          .getWorkerPools()
          .get(0)
          .getAutoscalingSettings()
          .getAlgorithm());
  assertEquals(
      42,
      job
          .getEnvironment()
          .getWorkerPools()
          .get(0)
          .getAutoscalingSettings()
          .getMaxNumWorkers()
          .intValue());
}
 

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
@Test
public void testNetworkConfigMissing() throws IOException {
  DataflowPipelineOptions options = buildPipelineOptions();

  Pipeline p = buildPipeline(options);
  p.traverseTopologically(new RecordingPipelineVisitor());
  Job job =
      DataflowPipelineTranslator.fromOptions(options)
          .translate(
              p, DataflowRunner.fromOptions(options), Collections.<DataflowPackage>emptyList())
          .getJob();

  assertEquals(1, job.getEnvironment().getWorkerPools().size());
  assertNull(job.getEnvironment().getWorkerPools().get(0).getNetwork());
}
 

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
public SamzaPipelineSpecification translate(Pipeline pipeline) {
  pipeline.traverseTopologically(this);
  return samzaPipelineSpec;
}
 

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
@Override
public DirectPipelineResult run(Pipeline originalPipeline) {
  Pipeline pipeline;
  try {
    RunnerApi.Pipeline protoPipeline = PipelineTranslation.toProto(originalPipeline);
    pipeline = PipelineTranslation.fromProto(protoPipeline);
  } catch (IOException exception) {
    throw new RuntimeException("Error preparing pipeline for direct execution.", exception);
  }
  pipeline.replaceAll(defaultTransformOverrides());
  MetricsEnvironment.setMetricsSupported(true);
  DirectGraphVisitor graphVisitor = new DirectGraphVisitor();
  pipeline.traverseTopologically(graphVisitor);

  @SuppressWarnings("rawtypes")
  KeyedPValueTrackingVisitor keyedPValueVisitor = KeyedPValueTrackingVisitor.create();
  pipeline.traverseTopologically(keyedPValueVisitor);

  DisplayDataValidator.validatePipeline(pipeline);
  DisplayDataValidator.validateOptions(getPipelineOptions());

  DirectGraph graph = graphVisitor.getGraph();
  EvaluationContext context =
      EvaluationContext.create(
          getPipelineOptions(),
          clockSupplier.get(),
          Enforcement.bundleFactoryFor(enabledEnforcements, graph),
          graph,
          keyedPValueVisitor.getKeyedPValues());

  TransformEvaluatorRegistry registry = TransformEvaluatorRegistry.defaultRegistry(context);
  PipelineExecutor executor =
      ExecutorServiceParallelExecutor.create(
          options.getTargetParallelism(),
          registry,
          Enforcement.defaultModelEnforcements(enabledEnforcements),
          context);
  executor.start(graph, RootProviderRegistry.defaultRegistry(context));

  DirectPipelineResult result = new DirectPipelineResult(executor, context);
  if (options.isBlockOnRun()) {
    try {
      result.waitUntilFinish();
    } catch (UserCodeException userException) {
      throw new PipelineExecutionException(userException.getCause());
    } catch (Throwable t) {
      if (t instanceof RuntimeException) {
        throw (RuntimeException) t;
      }
      throw new RuntimeException(t);
    }
  }
  return result;
}