Java Pipeline.replaceAll方法代码示例

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
/**
 * Depending on if the job is a Streaming or a Batch one, this method creates
 * the necessary execution environment and pipeline translator, and translates
 * the {@link org.apache.beam.sdk.values.PCollection} program into
 * a {@link org.apache.flink.api.java.DataSet}
 * or {@link org.apache.flink.streaming.api.datastream.DataStream} one.
 * */
public void translate(FlinkRunner flinkRunner, Pipeline pipeline) {
  this.flinkBatchEnv = null;
  this.flinkStreamEnv = null;

  pipeline.replaceAll(FlinkTransformOverrides.getDefaultOverrides(options.isStreaming()));

  PipelineTranslationOptimizer optimizer =
      new PipelineTranslationOptimizer(TranslationMode.BATCH, options);

  optimizer.translate(pipeline);
  TranslationMode translationMode = optimizer.getTranslationMode();

  FlinkPipelineTranslator translator;
  if (translationMode == TranslationMode.STREAMING) {
    this.flinkStreamEnv = createStreamExecutionEnvironment();
    translator = new FlinkStreamingPipelineTranslator(flinkRunner, flinkStreamEnv, options);
  } else {
    this.flinkBatchEnv = createBatchExecutionEnvironment();
    translator = new FlinkBatchPipelineTranslator(flinkBatchEnv, options);
  }

  translator.translate(pipeline);
}
 

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
@VisibleForTesting
void adaptBoundedReads(Pipeline pipeline) {
  pipeline.replaceAll(
      Collections.singletonList(
          PTransformOverride.of(
              PTransformMatchers.classEqualTo(BoundedReadFromUnboundedSource.class),
              new AdaptedBoundedAsUnbounded.Factory())));
}
 

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
@VisibleForTesting
protected void replaceTransforms(Pipeline pipeline) {
  boolean streaming = options.isStreaming() || containsUnboundedPCollection(pipeline);
  // Ensure all outputs of all reads are consumed before potentially replacing any
  // Read PTransforms
  UnconsumedReads.ensureAllReadsConsumed(pipeline);
  pipeline.replaceAll(getOverrides(streaming));
}
 

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
public void translate(Pipeline pipeline) {
  List<PTransformOverride> overrides =
      ImmutableList.<PTransformOverride>builder()
          .add(PTransformOverride.of(
              PTransformMatchers.classEqualTo(View.CreatePCollectionView.class),
              new CreateStreamingGearpumpView.Factory()))
          .build();

  pipeline.replaceAll(overrides);
  pipeline.traverseTopologically(this);
}
 

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
@Override
public DirectPipelineResult run(Pipeline originalPipeline) {
  Pipeline pipeline;
  try {
    RunnerApi.Pipeline protoPipeline = PipelineTranslation.toProto(originalPipeline);
    pipeline = PipelineTranslation.fromProto(protoPipeline);
  } catch (IOException exception) {
    throw new RuntimeException("Error preparing pipeline for direct execution.", exception);
  }
  pipeline.replaceAll(defaultTransformOverrides());
  MetricsEnvironment.setMetricsSupported(true);
  DirectGraphVisitor graphVisitor = new DirectGraphVisitor();
  pipeline.traverseTopologically(graphVisitor);

  @SuppressWarnings("rawtypes")
  KeyedPValueTrackingVisitor keyedPValueVisitor = KeyedPValueTrackingVisitor.create();
  pipeline.traverseTopologically(keyedPValueVisitor);

  DisplayDataValidator.validatePipeline(pipeline);
  DisplayDataValidator.validateOptions(getPipelineOptions());

  DirectGraph graph = graphVisitor.getGraph();
  EvaluationContext context =
      EvaluationContext.create(
          getPipelineOptions(),
          clockSupplier.get(),
          Enforcement.bundleFactoryFor(enabledEnforcements, graph),
          graph,
          keyedPValueVisitor.getKeyedPValues());

  TransformEvaluatorRegistry registry = TransformEvaluatorRegistry.defaultRegistry(context);
  PipelineExecutor executor =
      ExecutorServiceParallelExecutor.create(
          options.getTargetParallelism(),
          registry,
          Enforcement.defaultModelEnforcements(enabledEnforcements),
          context);
  executor.start(graph, RootProviderRegistry.defaultRegistry(context));

  DirectPipelineResult result = new DirectPipelineResult(executor, context);
  if (options.isBlockOnRun()) {
    try {
      result.waitUntilFinish();
    } catch (UserCodeException userException) {
      throw new PipelineExecutionException(userException.getCause());
    } catch (Throwable t) {
      if (t instanceof RuntimeException) {
        throw (RuntimeException) t;
      }
      throw new RuntimeException(t);
    }
  }
  return result;
}
 

import org.apache.beam.sdk.Pipeline; //导入方法依赖的package包/类
public static void performDirectOverrides(Pipeline p) {
  p.replaceAll(
      DirectRunner.fromOptions(PipelineOptionsFactory.create().as(DirectOptions.class))
          .defaultTransformOverrides());
}