Java Read.Unbounded方法代码示例

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
private static <T> TransformEvaluator<Read.Unbounded<T>> readUnbounded() {
  return new TransformEvaluator<Read.Unbounded<T>>() {
    @Override
    public void evaluate(Read.Unbounded<T> transform, EvaluationContext context) {
      final String stepName = context.getCurrentTransform().getFullName();
      context.putDataset(
          transform,
          SparkUnboundedSource.read(
              context.getStreamingContext(),
              context.getSerializableOptions(),
              transform.getSource(),
              stepName));
    }

    @Override
    public String toNativeString() {
      return "streamingContext.<readFrom(<source>)>()";
    }
  };
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
public void testTranslate() {
  ReadUnboundedTranslator translator = new ReadUnboundedTranslator();
  GearpumpPipelineOptions options =
      PipelineOptionsFactory.create().as(GearpumpPipelineOptions.class);
  Read.Unbounded transform = mock(Read.Unbounded.class);
  UnboundedSource source = mock(UnboundedSource.class);
  when(transform.getSource()).thenReturn(source);

  TranslationContext translationContext = mock(TranslationContext.class);
  when(translationContext.getPipelineOptions()).thenReturn(options);

  JavaStream stream = mock(JavaStream.class);
  PValue mockOutput = mock(PValue.class);
  when(translationContext.getOutput()).thenReturn(mockOutput);
  when(translationContext.getSourceStream(any(DataSource.class))).thenReturn(stream);

  translator.translate(transform, translationContext);
  verify(translationContext).getSourceStream(argThat(new UnboundedSourceWrapperMatcher()));
  verify(translationContext).setOutputStream(mockOutput, stream);
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public void visitPrimitiveTransform(TransformTreeNode node) {
  Class<? extends PTransform> transformClass = node.getTransform().getClass();
  if (transformClass == Read.Unbounded.class) {
    streaming = true;
  }
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public void translate(Read.Unbounded<T> transform, TranslationContext context) {
  UnboundedSource<T, ?> unboundedSource = transform.getSource();
  ApexReadUnboundedInputOperator<T, ?> operator = new ApexReadUnboundedInputOperator<>(
      unboundedSource, context.getPipelineOptions());
  context.addOperator(operator, operator.output);
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public FunctionSpec translate(
    AppliedPTransform<?, ?, Read.Unbounded<?>> transform, SdkComponents components) {
  ReadPayload payload = toProto(transform.getTransform());
  return RunnerApi.FunctionSpec.newBuilder()
      .setUrn(getUrn(transform.getTransform()))
      .setPayload(payload.toByteString())
      .build();
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Test
public void testToFromProtoUnbounded() throws Exception {
  assumeThat(source, instanceOf(UnboundedSource.class));
  UnboundedSource<?, ?> unboundedSource = (UnboundedSource<?, ?>) this.source;
  Read.Unbounded<?> unboundedRead = Read.from(unboundedSource);
  ReadPayload payload = ReadTranslation.toProto(unboundedRead);
  assertThat(payload.getIsBounded(), equalTo(RunnerApi.IsBounded.Enum.UNBOUNDED));
  UnboundedSource<?, ?> deserializedSource = ReadTranslation.unboundedSourceFromProto(payload);
  assertThat(deserializedSource, Matchers.<Source<?>>equalTo(source));
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public String toString() {
  try {
    Class<? extends PTransform> transformClass = transform.getClass();
    if (node.getFullName().equals("KafkaIO.Read")) {
      return "KafkaUtils.createDirectStream(...)";
    }
    if (composite) {
      return "_.<" + transformClass.getName() + ">";
    }
    String transformString = transformEvaluator.toNativeString();
    if (transformString.contains("<fn>")) {
      transformString = replaceFnString(transformClass, transformString, "fn");
    } else if (transformString.contains("<windowFn>")) {
      transformString = replaceFnString(transformClass, transformString, "windowFn");
    } else if (transformString.contains("<source>")) {
      String sourceName = "...";
      if (transform instanceof Read.Bounded) {
        sourceName = ((Read.Bounded<?>) transform).getSource().getClass().getName();
      } else if (transform instanceof Read.Unbounded) {
        sourceName = ((Read.Unbounded<?>) transform).getSource().getClass().getName();
      }
      transformString = transformString.replace("<source>", sourceName);
    }
    if (transformString.startsWith("sparkContext")
        || transformString.startsWith("streamingContext")) {
      return transformString;
    }
    return "_." + transformString;
  } catch (
      NoSuchMethodException
          | InvocationTargetException
          | IllegalAccessException
          | NoSuchFieldException e) {
    return "<FailedTranslation>";
  }
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public void visitPrimitiveTransform(TransformHierarchy.Node node) {
  Class<? extends PTransform> transformClass = node.getTransform().getClass();
  if (transformClass == Read.Unbounded.class) {
    LOG.info("Found {}. Switching to streaming execution.", transformClass);
    translationMode = TranslationMode.STREAMING;
  }
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public void translate(Read.Unbounded<T> transform, TranslationContext context) {
  UnboundedSource<T, ?> unboundedSource = transform.getSource();
  UnboundedSourceWrapper<T, ?> unboundedSourceWrapper = new UnboundedSourceWrapper<>(
      unboundedSource, context.getPipelineOptions());
  JavaStream<WindowedValue<T>> sourceStream = context.getSourceStream(unboundedSourceWrapper);

  context.setOutputStream(context.getOutput(), sourceStream);
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
public static ReadPayload toProto(Read.Unbounded<?> read) {
  return ReadPayload.newBuilder()
      .setIsBounded(IsBounded.Enum.UNBOUNDED)
      .setSource(toProto(read.getSource()))
      .build();
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public String getUrn(Read.Unbounded<?> transform) {
  return PTransformTranslation.READ_TRANSFORM_URN;
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
private static AppliedPTransform<?, ?, ?> read(Pipeline pipeline) {
  Read.Unbounded<Long> transform = Read.from(CountingSource.unbounded());
  PCollection<Long> pcollection = pipeline.apply(transform);
  return AppliedPTransform.<PBegin, PCollection<Long>, Read.Unbounded<Long>>of(
      "ReadTheCount", pipeline.begin().expand(), pcollection.expand(), transform, pipeline);
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public void translateNode(
    Read.Unbounded<T> transform,
    FlinkStreamingTranslationContext context) {
  PCollection<T> output = context.getOutput(transform);

  DataStream<WindowedValue<T>> source;
  DataStream<WindowedValue<ValueWithRecordId<T>>> nonDedupSource;
  TypeInformation<WindowedValue<T>> outputTypeInfo =
      context.getTypeInfo(context.getOutput(transform));

  Coder<T> coder = context.getOutput(transform).getCoder();

  TypeInformation<WindowedValue<ValueWithRecordId<T>>> withIdTypeInfo =
      new CoderTypeInformation<>(WindowedValue.getFullCoder(
          ValueWithRecordId.ValueWithRecordIdCoder.of(coder),
          output.getWindowingStrategy().getWindowFn().windowCoder()));

  try {

    UnboundedSourceWrapper<T, ?> sourceWrapper =
        new UnboundedSourceWrapper<>(
            context.getCurrentTransform().getFullName(),
            context.getPipelineOptions(),
            transform.getSource(),
            context.getExecutionEnvironment().getParallelism());
    nonDedupSource = context
        .getExecutionEnvironment()
        .addSource(sourceWrapper).name(transform.getName()).returns(withIdTypeInfo);

    if (transform.getSource().requiresDeduping()) {
      source = nonDedupSource.keyBy(
          new ValueWithRecordIdKeySelector<T>())
          .transform("deduping", outputTypeInfo, new DedupingOperator<T>());
    } else {
      source = nonDedupSource.flatMap(new StripIdsMap<T>()).returns(outputTypeInfo);
    }
  } catch (Exception e) {
    throw new RuntimeException(
        "Error while translating UnboundedSource: " + transform.getSource(), e);
  }

  context.setOutputDataStream(output, source);
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform(
    AppliedPTransform<PBegin, PCollection<T>, Read.Unbounded<T>> transform) {
  return PTransformReplacement.of(
      transform.getPipeline().begin(), new StreamingUnboundedRead<>(transform.getTransform()));
}
 

import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
public StreamingUnboundedRead(Read.Unbounded<T> transform) {
  this.source = transform.getSource();
}