本文整理汇总了Java中org.apache.beam.sdk.io.Read.Unbounded方法的典型用法代码示例。如果您正苦于以下问题:Java Read.Unbounded方法的具体用法?Java Read.Unbounded怎么用?Java Read.Unbounded使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.beam.sdk.io.Read
的用法示例。
在下文中一共展示了Read.Unbounded方法的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: readUnbounded
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
private static <T> TransformEvaluator<Read.Unbounded<T>> readUnbounded() {
return new TransformEvaluator<Read.Unbounded<T>>() {
@Override
public void evaluate(Read.Unbounded<T> transform, EvaluationContext context) {
final String stepName = context.getCurrentTransform().getFullName();
context.putDataset(
transform,
SparkUnboundedSource.read(
context.getStreamingContext(),
context.getSerializableOptions(),
transform.getSource(),
stepName));
}
@Override
public String toNativeString() {
return "streamingContext.<readFrom(<source>)>()";
}
};
}
示例2: testTranslate
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
public void testTranslate() {
ReadUnboundedTranslator translator = new ReadUnboundedTranslator();
GearpumpPipelineOptions options =
PipelineOptionsFactory.create().as(GearpumpPipelineOptions.class);
Read.Unbounded transform = mock(Read.Unbounded.class);
UnboundedSource source = mock(UnboundedSource.class);
when(transform.getSource()).thenReturn(source);
TranslationContext translationContext = mock(TranslationContext.class);
when(translationContext.getPipelineOptions()).thenReturn(options);
JavaStream stream = mock(JavaStream.class);
PValue mockOutput = mock(PValue.class);
when(translationContext.getOutput()).thenReturn(mockOutput);
when(translationContext.getSourceStream(any(DataSource.class))).thenReturn(stream);
translator.translate(transform, translationContext);
verify(translationContext).getSourceStream(argThat(new UnboundedSourceWrapperMatcher()));
verify(translationContext).setOutputStream(mockOutput, stream);
}
示例3: visitPrimitiveTransform
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public void visitPrimitiveTransform(TransformTreeNode node) {
Class<? extends PTransform> transformClass = node.getTransform().getClass();
if (transformClass == Read.Unbounded.class) {
streaming = true;
}
}
示例4: translate
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public void translate(Read.Unbounded<T> transform, TranslationContext context) {
UnboundedSource<T, ?> unboundedSource = transform.getSource();
ApexReadUnboundedInputOperator<T, ?> operator = new ApexReadUnboundedInputOperator<>(
unboundedSource, context.getPipelineOptions());
context.addOperator(operator, operator.output);
}
示例5: translate
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public FunctionSpec translate(
AppliedPTransform<?, ?, Read.Unbounded<?>> transform, SdkComponents components) {
ReadPayload payload = toProto(transform.getTransform());
return RunnerApi.FunctionSpec.newBuilder()
.setUrn(getUrn(transform.getTransform()))
.setPayload(payload.toByteString())
.build();
}
示例6: testToFromProtoUnbounded
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Test
public void testToFromProtoUnbounded() throws Exception {
assumeThat(source, instanceOf(UnboundedSource.class));
UnboundedSource<?, ?> unboundedSource = (UnboundedSource<?, ?>) this.source;
Read.Unbounded<?> unboundedRead = Read.from(unboundedSource);
ReadPayload payload = ReadTranslation.toProto(unboundedRead);
assertThat(payload.getIsBounded(), equalTo(RunnerApi.IsBounded.Enum.UNBOUNDED));
UnboundedSource<?, ?> deserializedSource = ReadTranslation.unboundedSourceFromProto(payload);
assertThat(deserializedSource, Matchers.<Source<?>>equalTo(source));
}
示例7: toString
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public String toString() {
try {
Class<? extends PTransform> transformClass = transform.getClass();
if (node.getFullName().equals("KafkaIO.Read")) {
return "KafkaUtils.createDirectStream(...)";
}
if (composite) {
return "_.<" + transformClass.getName() + ">";
}
String transformString = transformEvaluator.toNativeString();
if (transformString.contains("<fn>")) {
transformString = replaceFnString(transformClass, transformString, "fn");
} else if (transformString.contains("<windowFn>")) {
transformString = replaceFnString(transformClass, transformString, "windowFn");
} else if (transformString.contains("<source>")) {
String sourceName = "...";
if (transform instanceof Read.Bounded) {
sourceName = ((Read.Bounded<?>) transform).getSource().getClass().getName();
} else if (transform instanceof Read.Unbounded) {
sourceName = ((Read.Unbounded<?>) transform).getSource().getClass().getName();
}
transformString = transformString.replace("<source>", sourceName);
}
if (transformString.startsWith("sparkContext")
|| transformString.startsWith("streamingContext")) {
return transformString;
}
return "_." + transformString;
} catch (
NoSuchMethodException
| InvocationTargetException
| IllegalAccessException
| NoSuchFieldException e) {
return "<FailedTranslation>";
}
}
示例8: visitPrimitiveTransform
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public void visitPrimitiveTransform(TransformHierarchy.Node node) {
Class<? extends PTransform> transformClass = node.getTransform().getClass();
if (transformClass == Read.Unbounded.class) {
LOG.info("Found {}. Switching to streaming execution.", transformClass);
translationMode = TranslationMode.STREAMING;
}
}
示例9: translate
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public void translate(Read.Unbounded<T> transform, TranslationContext context) {
UnboundedSource<T, ?> unboundedSource = transform.getSource();
UnboundedSourceWrapper<T, ?> unboundedSourceWrapper = new UnboundedSourceWrapper<>(
unboundedSource, context.getPipelineOptions());
JavaStream<WindowedValue<T>> sourceStream = context.getSourceStream(unboundedSourceWrapper);
context.setOutputStream(context.getOutput(), sourceStream);
}
示例10: toProto
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
public static ReadPayload toProto(Read.Unbounded<?> read) {
return ReadPayload.newBuilder()
.setIsBounded(IsBounded.Enum.UNBOUNDED)
.setSource(toProto(read.getSource()))
.build();
}
示例11: getUrn
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public String getUrn(Read.Unbounded<?> transform) {
return PTransformTranslation.READ_TRANSFORM_URN;
}
示例12: read
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
private static AppliedPTransform<?, ?, ?> read(Pipeline pipeline) {
Read.Unbounded<Long> transform = Read.from(CountingSource.unbounded());
PCollection<Long> pcollection = pipeline.apply(transform);
return AppliedPTransform.<PBegin, PCollection<Long>, Read.Unbounded<Long>>of(
"ReadTheCount", pipeline.begin().expand(), pcollection.expand(), transform, pipeline);
}
示例13: translateNode
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public void translateNode(
Read.Unbounded<T> transform,
FlinkStreamingTranslationContext context) {
PCollection<T> output = context.getOutput(transform);
DataStream<WindowedValue<T>> source;
DataStream<WindowedValue<ValueWithRecordId<T>>> nonDedupSource;
TypeInformation<WindowedValue<T>> outputTypeInfo =
context.getTypeInfo(context.getOutput(transform));
Coder<T> coder = context.getOutput(transform).getCoder();
TypeInformation<WindowedValue<ValueWithRecordId<T>>> withIdTypeInfo =
new CoderTypeInformation<>(WindowedValue.getFullCoder(
ValueWithRecordId.ValueWithRecordIdCoder.of(coder),
output.getWindowingStrategy().getWindowFn().windowCoder()));
try {
UnboundedSourceWrapper<T, ?> sourceWrapper =
new UnboundedSourceWrapper<>(
context.getCurrentTransform().getFullName(),
context.getPipelineOptions(),
transform.getSource(),
context.getExecutionEnvironment().getParallelism());
nonDedupSource = context
.getExecutionEnvironment()
.addSource(sourceWrapper).name(transform.getName()).returns(withIdTypeInfo);
if (transform.getSource().requiresDeduping()) {
source = nonDedupSource.keyBy(
new ValueWithRecordIdKeySelector<T>())
.transform("deduping", outputTypeInfo, new DedupingOperator<T>());
} else {
source = nonDedupSource.flatMap(new StripIdsMap<T>()).returns(outputTypeInfo);
}
} catch (Exception e) {
throw new RuntimeException(
"Error while translating UnboundedSource: " + transform.getSource(), e);
}
context.setOutputDataStream(output, source);
}
示例14: getReplacementTransform
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform(
AppliedPTransform<PBegin, PCollection<T>, Read.Unbounded<T>> transform) {
return PTransformReplacement.of(
transform.getPipeline().begin(), new StreamingUnboundedRead<>(transform.getTransform()));
}
示例15: StreamingUnboundedRead
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
public StreamingUnboundedRead(Read.Unbounded<T> transform) {
this.source = transform.getSource();
}