本文整理汇总了Java中org.apache.beam.sdk.io.Read.Bounded方法的典型用法代码示例。如果您正苦于以下问题:Java Read.Bounded方法的具体用法?Java Read.Bounded怎么用?Java Read.Bounded使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.beam.sdk.io.Read
的用法示例。
在下文中一共展示了Read.Bounded方法的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: readBounded
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
private static <T> TransformEvaluator<Read.Bounded<T>> readBounded() {
return new TransformEvaluator<Read.Bounded<T>>() {
@Override
public void evaluate(Read.Bounded<T> transform, EvaluationContext context) {
String stepName = context.getCurrentTransform().getFullName();
final JavaSparkContext jsc = context.getSparkContext();
// create an RDD from a BoundedSource.
JavaRDD<WindowedValue<T>> input =
new SourceRDD.Bounded<>(
jsc.sc(), transform.getSource(), context.getSerializableOptions(), stepName)
.toJavaRDD();
// cache to avoid re-evaluation of the source by Spark's lazy DAG evaluation.
context.putDataset(transform, new BoundedDataset<>(input), true);
}
@Override
public String toNativeString() {
return "sparkContext.<readFrom(<source>)>()";
}
};
}
示例2: testTranslate
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Test
@SuppressWarnings({"rawtypes", "unchecked"})
public void testTranslate() {
ReadBoundedTranslator translator = new ReadBoundedTranslator();
GearpumpPipelineOptions options =
PipelineOptionsFactory.create().as(GearpumpPipelineOptions.class);
Read.Bounded transform = mock(Read.Bounded.class);
BoundedSource source = mock(BoundedSource.class);
when(transform.getSource()).thenReturn(source);
TranslationContext translationContext = mock(TranslationContext.class);
when(translationContext.getPipelineOptions()).thenReturn(options);
JavaStream stream = mock(JavaStream.class);
PValue mockOutput = mock(PValue.class);
when(translationContext.getOutput()).thenReturn(mockOutput);
when(translationContext.getSourceStream(any(DataSource.class))).thenReturn(stream);
translator.translate(transform, translationContext);
verify(translationContext).getSourceStream(argThat(new BoundedSourceWrapperMatcher()));
verify(translationContext).setOutputStream(mockOutput, stream);
}
示例3: translate
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public void translate(Read.Bounded<T> transform, TranslationContext context) {
// TODO: adapter is visibleForTesting
BoundedToUnboundedSourceAdapter unboundedSource = new BoundedToUnboundedSourceAdapter<>(
transform.getSource());
ApexReadUnboundedInputOperator<T, ?> operator = new ApexReadUnboundedInputOperator<>(
unboundedSource, true, context.getPipelineOptions());
context.addOperator(operator, operator.output);
}
示例4: translate
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public FunctionSpec translate(
AppliedPTransform<?, ?, Read.Bounded<?>> transform, SdkComponents components) {
ReadPayload payload = toProto(transform.getTransform());
return RunnerApi.FunctionSpec.newBuilder()
.setUrn(getUrn(transform.getTransform()))
.setPayload(payload.toByteString())
.build();
}
示例5: testToFromProtoBounded
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Test
public void testToFromProtoBounded() throws Exception {
// TODO: Split into two tests.
assumeThat(source, instanceOf(BoundedSource.class));
BoundedSource<?> boundedSource = (BoundedSource<?>) this.source;
Read.Bounded<?> boundedRead = Read.from(boundedSource);
ReadPayload payload = ReadTranslation.toProto(boundedRead);
assertThat(payload.getIsBounded(), equalTo(RunnerApi.IsBounded.Enum.BOUNDED));
BoundedSource<?> deserializedSource = ReadTranslation.boundedSourceFromProto(payload);
assertThat(deserializedSource, Matchers.<Source<?>>equalTo(source));
}
示例6: toString
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public String toString() {
try {
Class<? extends PTransform> transformClass = transform.getClass();
if (node.getFullName().equals("KafkaIO.Read")) {
return "KafkaUtils.createDirectStream(...)";
}
if (composite) {
return "_.<" + transformClass.getName() + ">";
}
String transformString = transformEvaluator.toNativeString();
if (transformString.contains("<fn>")) {
transformString = replaceFnString(transformClass, transformString, "fn");
} else if (transformString.contains("<windowFn>")) {
transformString = replaceFnString(transformClass, transformString, "windowFn");
} else if (transformString.contains("<source>")) {
String sourceName = "...";
if (transform instanceof Read.Bounded) {
sourceName = ((Read.Bounded<?>) transform).getSource().getClass().getName();
} else if (transform instanceof Read.Unbounded) {
sourceName = ((Read.Unbounded<?>) transform).getSource().getClass().getName();
}
transformString = transformString.replace("<source>", sourceName);
}
if (transformString.startsWith("sparkContext")
|| transformString.startsWith("streamingContext")) {
return transformString;
}
return "_." + transformString;
} catch (
NoSuchMethodException
| InvocationTargetException
| IllegalAccessException
| NoSuchFieldException e) {
return "<FailedTranslation>";
}
}
示例7: translateNode
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public void translateNode(
Read.Bounded<T> transform,
FlinkStreamingTranslationContext context) {
PCollection<T> output = context.getOutput(transform);
TypeInformation<WindowedValue<T>> outputTypeInfo =
context.getTypeInfo(context.getOutput(transform));
DataStream<WindowedValue<T>> source;
try {
BoundedSourceWrapper<T> sourceWrapper =
new BoundedSourceWrapper<>(
context.getCurrentTransform().getFullName(),
context.getPipelineOptions(),
transform.getSource(),
context.getExecutionEnvironment().getParallelism());
source = context
.getExecutionEnvironment()
.addSource(sourceWrapper).name(transform.getName()).returns(outputTypeInfo);
} catch (Exception e) {
throw new RuntimeException(
"Error while translating BoundedSource: " + transform.getSource(), e);
}
context.setOutputDataStream(output, source);
}
示例8: translate
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public void translate(Read.Bounded<T> transform, TranslationContext context) {
BoundedSource<T> boundedSource = transform.getSource();
BoundedSourceWrapper<T> sourceWrapper = new BoundedSourceWrapper<>(boundedSource,
context.getPipelineOptions());
JavaStream<WindowedValue<T>> sourceStream = context.getSourceStream(sourceWrapper);
context.setOutputStream(context.getOutput(), sourceStream);
}
示例9: replaceSucceeds
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Test
public void replaceSucceeds() {
PTransform<?, ?> enclosingPT =
new PTransform<PInput, POutput>() {
@Override
public POutput expand(PInput input) {
return PDone.in(input.getPipeline());
}
};
TransformHierarchy.Node enclosing =
hierarchy.pushNode("Enclosing", PBegin.in(pipeline), enclosingPT);
Create.Values<Long> originalTransform = Create.of(1L);
TransformHierarchy.Node original =
hierarchy.pushNode("Create", PBegin.in(pipeline), originalTransform);
assertThat(hierarchy.getCurrent(), equalTo(original));
PCollection<Long> originalOutput = pipeline.apply(originalTransform);
hierarchy.setOutput(originalOutput);
hierarchy.popNode();
assertThat(original.finishedSpecifying, is(true));
hierarchy.setOutput(PDone.in(pipeline));
hierarchy.popNode();
assertThat(hierarchy.getCurrent(), not(equalTo(enclosing)));
Read.Bounded<Long> replacementTransform = Read.from(CountingSource.upTo(1L));
PCollection<Long> replacementOutput = pipeline.apply(replacementTransform);
Node replacement = hierarchy.replaceNode(original, PBegin.in(pipeline), replacementTransform);
assertThat(hierarchy.getCurrent(), equalTo(replacement));
hierarchy.setOutput(replacementOutput);
TaggedPValue taggedReplacement = TaggedPValue.ofExpandedValue(replacementOutput);
Map<PValue, ReplacementOutput> replacementOutputs =
Collections.<PValue, ReplacementOutput>singletonMap(
replacementOutput,
ReplacementOutput.of(
TaggedPValue.ofExpandedValue(originalOutput),
taggedReplacement));
hierarchy.replaceOutputs(replacementOutputs);
assertThat(replacement.getInputs(), equalTo(original.getInputs()));
assertThat(replacement.getEnclosingNode(), equalTo(original.getEnclosingNode()));
assertThat(replacement.getEnclosingNode(), equalTo(enclosing));
assertThat(
replacement.getTransform(), Matchers.<PTransform<?, ?>>equalTo(replacementTransform));
// THe tags of the replacement transform are matched to the appropriate PValues of the original
assertThat(
replacement.getOutputs().keySet(),
Matchers.<TupleTag<?>>contains(taggedReplacement.getTag()));
assertThat(replacement.getOutputs().values(), Matchers.<PValue>contains(originalOutput));
hierarchy.popNode();
}
示例10: toProto
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
public static ReadPayload toProto(Read.Bounded<?> read) {
return ReadPayload.newBuilder()
.setIsBounded(IsBounded.Enum.BOUNDED)
.setSource(toProto(read.getSource()))
.build();
}
示例11: getUrn
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public String getUrn(Read.Bounded<?> transform) {
return PTransformTranslation.READ_TRANSFORM_URN;
}
示例12: translate
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public void translate(Read.Bounded<?> transform, TranslationContext context) {
translateReadHelper(transform.getSource(), transform, context);
}
示例13: getReplacementTransform
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
@Override
public PTransformReplacement<PBegin, PCollection<T>> getReplacementTransform(
AppliedPTransform<PBegin, PCollection<T>, Read.Bounded<T>> transform) {
return PTransformReplacement.of(
transform.getPipeline().begin(), new StreamingBoundedRead<>(transform.getTransform()));
}
示例14: StreamingBoundedRead
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
public StreamingBoundedRead(Read.Bounded<T> transform) {
this.source = transform.getSource();
}
示例15: FnApiBoundedRead
import org.apache.beam.sdk.io.Read; //导入方法依赖的package包/类
public FnApiBoundedRead(Read.Bounded<T> transform) {
this.source = transform.getSource();
}