Java Read.Bounded方法代码示例

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
private static <T> TransformEvaluator<Read.Bounded<T>> readBounded() {
  return new TransformEvaluator<Read.Bounded<T>>() {
    @Override
    public void evaluate(Read.Bounded<T> transform, EvaluationContext context) {
      String stepName = context.getCurrentTransform().getFullName();
      final JavaSparkContext jsc = context.getSparkContext();
      // create an RDD from a BoundedSource.
      JavaRDD<WindowedValue<T>> input =
          new SourceRDD.Bounded<>(
                  jsc.sc(), transform.getSource(), context.getSerializableOptions(), stepName)
              .toJavaRDD();
      // cache to avoid re-evaluation of the source by Spark's lazy DAG evaluation.
      context.putDataset(transform, new BoundedDataset<>(input), true);
    }

    @Override
    public String toNativeString() {
      return "sparkContext.<readFrom(<source>)>()";
    }
  };
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
public void testTranslate() {
  ReadBoundedTranslator translator = new ReadBoundedTranslator();
  GearpumpPipelineOptions options =
      PipelineOptionsFactory.create().as(GearpumpPipelineOptions.class);
  Read.Bounded transform = mock(Read.Bounded.class);
  BoundedSource source = mock(BoundedSource.class);
  when(transform.getSource()).thenReturn(source);

  TranslationContext translationContext = mock(TranslationContext.class);
  when(translationContext.getPipelineOptions()).thenReturn(options);

  JavaStream stream = mock(JavaStream.class);
  PValue mockOutput = mock(PValue.class);
  when(translationContext.getOutput()).thenReturn(mockOutput);
  when(translationContext.getSourceStream(any(DataSource.class))).thenReturn(stream);

  translator.translate(transform, translationContext);
  verify(translationContext).getSourceStream(argThat(new BoundedSourceWrapperMatcher()));
  verify(translationContext).setOutputStream(mockOutput, stream);
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public void translate(Read.Bounded<T> transform, TranslationContext context) {
  // TODO: adapter is visibleForTesting
  BoundedToUnboundedSourceAdapter unboundedSource = new BoundedToUnboundedSourceAdapter<>(
      transform.getSource());
  ApexReadUnboundedInputOperator<T, ?> operator = new ApexReadUnboundedInputOperator<>(
      unboundedSource, true, context.getPipelineOptions());
  context.addOperator(operator, operator.output);
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public FunctionSpec translate(
    AppliedPTransform<?, ?, Read.Bounded<?>> transform, SdkComponents components) {
  ReadPayload payload = toProto(transform.getTransform());
  return RunnerApi.FunctionSpec.newBuilder()
      .setUrn(getUrn(transform.getTransform()))
      .setPayload(payload.toByteString())
      .build();
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Test
public void testToFromProtoBounded() throws Exception {
  // TODO: Split into two tests.
  assumeThat(source, instanceOf(BoundedSource.class));
  BoundedSource<?> boundedSource = (BoundedSource<?>) this.source;
  Read.Bounded<?> boundedRead = Read.from(boundedSource);
  ReadPayload payload = ReadTranslation.toProto(boundedRead);
  assertThat(payload.getIsBounded(), equalTo(RunnerApi.IsBounded.Enum.BOUNDED));
  BoundedSource<?> deserializedSource = ReadTranslation.boundedSourceFromProto(payload);
  assertThat(deserializedSource, Matchers.<Source<?>>equalTo(source));
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public String toString() {
  try {
    Class<? extends PTransform> transformClass = transform.getClass();
    if (node.getFullName().equals("KafkaIO.Read")) {
      return "KafkaUtils.createDirectStream(...)";
    }
    if (composite) {
      return "_.<" + transformClass.getName() + ">";
    }
    String transformString = transformEvaluator.toNativeString();
    if (transformString.contains("<fn>")) {
      transformString = replaceFnString(transformClass, transformString, "fn");
    } else if (transformString.contains("<windowFn>")) {
      transformString = replaceFnString(transformClass, transformString, "windowFn");
    } else if (transformString.contains("<source>")) {
      String sourceName = "...";
      if (transform instanceof Read.Bounded) {
        sourceName = ((Read.Bounded<?>) transform).getSource().getClass().getName();
      } else if (transform instanceof Read.Unbounded) {
        sourceName = ((Read.Unbounded<?>) transform).getSource().getClass().getName();
      }
      transformString = transformString.replace("<source>", sourceName);
    }
    if (transformString.startsWith("sparkContext")
        || transformString.startsWith("streamingContext")) {
      return transformString;
    }
    return "_." + transformString;
  } catch (
      NoSuchMethodException
          | InvocationTargetException
          | IllegalAccessException
          | NoSuchFieldException e) {
    return "<FailedTranslation>";
  }
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public void translateNode(
    Read.Bounded<T> transform,
    FlinkStreamingTranslationContext context) {
  PCollection<T> output = context.getOutput(transform);

  TypeInformation<WindowedValue<T>> outputTypeInfo =
      context.getTypeInfo(context.getOutput(transform));


  DataStream<WindowedValue<T>> source;
  try {
    BoundedSourceWrapper<T> sourceWrapper =
        new BoundedSourceWrapper<>(
            context.getCurrentTransform().getFullName(),
            context.getPipelineOptions(),
            transform.getSource(),
            context.getExecutionEnvironment().getParallelism());
    source = context
        .getExecutionEnvironment()
        .addSource(sourceWrapper).name(transform.getName()).returns(outputTypeInfo);
  } catch (Exception e) {
    throw new RuntimeException(
        "Error while translating BoundedSource: " + transform.getSource(), e);
  }

  context.setOutputDataStream(output, source);
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public void translate(Read.Bounded<T> transform, TranslationContext context) {
  BoundedSource<T> boundedSource = transform.getSource();
  BoundedSourceWrapper<T> sourceWrapper = new BoundedSourceWrapper<>(boundedSource,
      context.getPipelineOptions());
  JavaStream<WindowedValue<T>> sourceStream = context.getSourceStream(sourceWrapper);

  context.setOutputStream(context.getOutput(), sourceStream);
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Test
public void replaceSucceeds() {
  PTransform<?, ?> enclosingPT =
      new PTransform<PInput, POutput>() {
        @Override
        public POutput expand(PInput input) {
          return PDone.in(input.getPipeline());
        }
      };

  TransformHierarchy.Node enclosing =
      hierarchy.pushNode("Enclosing", PBegin.in(pipeline), enclosingPT);

  Create.Values<Long> originalTransform = Create.of(1L);
  TransformHierarchy.Node original =
      hierarchy.pushNode("Create", PBegin.in(pipeline), originalTransform);
  assertThat(hierarchy.getCurrent(), equalTo(original));
  PCollection<Long> originalOutput = pipeline.apply(originalTransform);
  hierarchy.setOutput(originalOutput);
  hierarchy.popNode();
  assertThat(original.finishedSpecifying, is(true));
  hierarchy.setOutput(PDone.in(pipeline));
  hierarchy.popNode();

  assertThat(hierarchy.getCurrent(), not(equalTo(enclosing)));
  Read.Bounded<Long> replacementTransform = Read.from(CountingSource.upTo(1L));
  PCollection<Long> replacementOutput = pipeline.apply(replacementTransform);
  Node replacement = hierarchy.replaceNode(original, PBegin.in(pipeline), replacementTransform);
  assertThat(hierarchy.getCurrent(), equalTo(replacement));
  hierarchy.setOutput(replacementOutput);

  TaggedPValue taggedReplacement = TaggedPValue.ofExpandedValue(replacementOutput);
  Map<PValue, ReplacementOutput> replacementOutputs =
      Collections.<PValue, ReplacementOutput>singletonMap(
          replacementOutput,
          ReplacementOutput.of(
              TaggedPValue.ofExpandedValue(originalOutput),
              taggedReplacement));
  hierarchy.replaceOutputs(replacementOutputs);

  assertThat(replacement.getInputs(), equalTo(original.getInputs()));
  assertThat(replacement.getEnclosingNode(), equalTo(original.getEnclosingNode()));
  assertThat(replacement.getEnclosingNode(), equalTo(enclosing));
  assertThat(
      replacement.getTransform(), Matchers.<PTransform<?, ?>>equalTo(replacementTransform));
  // THe tags of the replacement transform are matched to the appropriate PValues of the original
  assertThat(
      replacement.getOutputs().keySet(),
      Matchers.<TupleTag<?>>contains(taggedReplacement.getTag()));
  assertThat(replacement.getOutputs().values(), Matchers.<PValue>contains(originalOutput));
  hierarchy.popNode();
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
public static ReadPayload toProto(Read.Bounded<?> read) {
  return ReadPayload.newBuilder()
      .setIsBounded(IsBounded.Enum.BOUNDED)
      .setSource(toProto(read.getSource()))
      .build();
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public String getUrn(Read.Bounded<?> transform) {
  return PTransformTranslation.READ_TRANSFORM_URN;
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public void translate(Read.Bounded<?> transform, TranslationContext context) {
  translateReadHelper(transform.getSource(), transform, context);
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform(
    AppliedPTransform<PBegin, PCollection<T>, Read.Bounded<T>> transform) {
  return PTransformReplacement.of(
      transform.getPipeline().begin(), new StreamingBoundedRead<>(transform.getTransform()));
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
public StreamingBoundedRead(Read.Bounded<T> transform) {
  this.source = transform.getSource();
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
public FnApiBoundedRead(Read.Bounded<T> transform) {
  this.source = transform.getSource();
}