Java Window.Assign方法代码示例

import org.apache.beam.sdk.transforms.windowing.Window; //导入方法依赖的package包/类
@Override
public void translate(Window.Assign<T> transform, TranslationContext context) {
  PCollection<T> output = context.getOutput();
  PCollection<T> input = context.getInput();

 if (transform.getWindowFn() == null) {
   // no work to do
   context.addAlias(output, input);
 } else {
    @SuppressWarnings("unchecked")
    WindowFn<T, BoundedWindow> windowFn = (WindowFn<T, BoundedWindow>) transform.getWindowFn();
    ApexProcessFnOperator<T> operator = ApexProcessFnOperator.assignWindows(windowFn,
        context.getPipelineOptions());
    context.addOperator(operator, operator.outputPort);
    context.addStream(context.getInput(), operator.inputPort);
 }

}
 

import org.apache.beam.sdk.transforms.windowing.Window; //导入方法依赖的package包/类
private static <T, W extends BoundedWindow> TransformEvaluator<Window.Assign<T>> window() {
  return new TransformEvaluator<Window.Assign<T>>() {
    @Override
    public void evaluate(Window.Assign<T> transform, EvaluationContext context) {
      @SuppressWarnings("unchecked")
      JavaRDD<WindowedValue<T>> inRDD =
          ((BoundedDataset<T>) context.borrowDataset(transform)).getRDD();

      if (TranslationUtils.skipAssignWindows(transform, context)) {
        context.putDataset(transform, new BoundedDataset<>(inRDD));
      } else {
        context.putDataset(transform, new BoundedDataset<>(
            inRDD.map(new SparkAssignWindowFn<>(transform.getWindowFn()))));
      }
    }

    @Override
    public String toNativeString() {
      return "map(new <windowFn>())";
    }
  };
}
 

import org.apache.beam.sdk.transforms.windowing.Window; //导入方法依赖的package包/类
@Override
public FunctionSpec translate(
    AppliedPTransform<?, ?, Window.Assign<?>> transform, SdkComponents components) {
  return FunctionSpec.newBuilder()
      .setUrn("urn:beam:transform:window:v1")
      .setPayload(
          WindowIntoTranslation.toProto(transform.getTransform(), components).toByteString())
      .build();
}
 

import org.apache.beam.sdk.transforms.windowing.Window; //导入方法依赖的package包/类
private static <T, W extends BoundedWindow> TransformEvaluator<Window.Assign<T>> window() {
  return new TransformEvaluator<Window.Assign<T>>() {
    @Override
    public void evaluate(final Window.Assign<T> transform, EvaluationContext context) {
      @SuppressWarnings("unchecked")
      UnboundedDataset<T> unboundedDataset =
          ((UnboundedDataset<T>) context.borrowDataset(transform));
      JavaDStream<WindowedValue<T>> dStream = unboundedDataset.getDStream();
      JavaDStream<WindowedValue<T>> outputStream;
      if (TranslationUtils.skipAssignWindows(transform, context)) {
        // do nothing.
        outputStream = dStream;
      } else {
        outputStream = dStream.transform(
            new Function<JavaRDD<WindowedValue<T>>, JavaRDD<WindowedValue<T>>>() {
          @Override
          public JavaRDD<WindowedValue<T>> call(JavaRDD<WindowedValue<T>> rdd) throws Exception {
            return rdd.map(new SparkAssignWindowFn<>(transform.getWindowFn()));
          }
        });
      }
      context.putDataset(transform,
          new UnboundedDataset<>(outputStream, unboundedDataset.getStreamSources()));
    }

    @Override
    public String toNativeString() {
      return "map(new <windowFn>())";
    }
  };
}
 

import org.apache.beam.sdk.transforms.windowing.Window; //导入方法依赖的package包/类
@Override
public void translateNode(
    Window.Assign<T> transform,
    FlinkStreamingTranslationContext context) {

  @SuppressWarnings("unchecked")
  WindowingStrategy<T, BoundedWindow> windowingStrategy =
      (WindowingStrategy<T, BoundedWindow>)
          context.getOutput(transform).getWindowingStrategy();

  TypeInformation<WindowedValue<T>> typeInfo =
      context.getTypeInfo(context.getOutput(transform));

  DataStream<WindowedValue<T>> inputDataStream =
      context.getInputDataStream(context.getInput(transform));

  WindowFn<T, ? extends BoundedWindow> windowFn = windowingStrategy.getWindowFn();

  FlinkAssignWindows<T, ? extends BoundedWindow> assignWindowsFunction =
      new FlinkAssignWindows<>(windowFn);

  SingleOutputStreamOperator<WindowedValue<T>> outputDataStream = inputDataStream
      .flatMap(assignWindowsFunction)
      .name(context.getOutput(transform).getName())
      .returns(typeInfo);

  context.setOutputDataStream(context.getOutput(transform), outputDataStream);
}
 

import org.apache.beam.sdk.transforms.windowing.Window; //导入方法依赖的package包/类
private <T> void translateHelper(Window.Assign<T> transform, TranslationContext context) {
  StepTranslationContext stepContext = context.addStep(transform, "Bucket");
  PCollection<T> input = context.getInput(transform);
  stepContext.addInput(PropertyNames.PARALLEL_INPUT, input);
  stepContext.addOutput(context.getOutput(transform));

  WindowingStrategy<?, ?> strategy = context.getOutput(transform).getWindowingStrategy();
  byte[] serializedBytes = serializeWindowingStrategy(strategy);
  String serializedJson = byteArrayToJsonString(serializedBytes);
  stepContext.addInput(PropertyNames.SERIALIZED_FN, serializedJson);
}
 

import org.apache.beam.sdk.transforms.windowing.Window; //导入方法依赖的package包/类
@Override
public void translate(Window.Assign<T> transform, TranslationContext context) {
  PCollection<T> input = (PCollection<T>) context.getInput();
  PCollection<T> output = (PCollection<T>) context.getOutput();
  JavaStream<WindowedValue<T>> inputStream = context.getInputStream(input);
  WindowingStrategy<?, ?> outputStrategy = output.getWindowingStrategy();
  WindowFn<T, BoundedWindow> windowFn = (WindowFn<T, BoundedWindow>) outputStrategy.getWindowFn();
  JavaStream<WindowedValue<T>> outputStream =
      inputStream
          .flatMap(new AssignWindows(windowFn), "assign_windows");

  context.setOutputStream(output, outputStream);
}
 

import org.apache.beam.sdk.transforms.windowing.Window; //导入方法依赖的package包/类
private <InputT> TransformEvaluator<InputT> createTransformEvaluator(
    AppliedPTransform<PCollection<InputT>, PCollection<InputT>, Window.Assign<InputT>>
        transform) {

  WindowFn<? super InputT, ?> fn = (WindowFn) WindowIntoTranslation.getWindowFn(transform);

  UncommittedBundle<InputT> outputBundle =
      evaluationContext.createBundle(
          (PCollection<InputT>) Iterables.getOnlyElement(transform.getOutputs().values()));
  if (fn == null) {
    return PassthroughTransformEvaluator.create(transform, outputBundle);
  }
  return new WindowIntoEvaluator<>(transform, fn, outputBundle);
}
 

import org.apache.beam.sdk.transforms.windowing.Window; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
public WindowIntoEvaluator(
    AppliedPTransform<PCollection<InputT>, PCollection<InputT>, Window.Assign<InputT>>
        transform,
    WindowFn<? super InputT, ?> windowFn,
    UncommittedBundle<InputT> outputBundle) {
  this.outputBundle = outputBundle;
  this.transform = transform;
  // Safe contravariant cast
  this.windowFn = (WindowFn<InputT, ?>) windowFn;
}
 

import org.apache.beam.sdk.transforms.windowing.Window; //导入方法依赖的package包/类
public static WindowIntoPayload toProto(Window.Assign<?> transform, SdkComponents components) {
  return WindowIntoPayload.newBuilder()
      .setWindowFn(WindowingStrategyTranslation.toProto(transform.getWindowFn(), components))
      .build();
}
 

import org.apache.beam.sdk.transforms.windowing.Window; //导入方法依赖的package包/类
@Override
public String getUrn(Window.Assign<?> transform) {
  return PTransformTranslation.WINDOW_TRANSFORM_URN;
}
 

import org.apache.beam.sdk.transforms.windowing.Window; //导入方法依赖的package包/类
@Override
public void translate(Window.Assign transform, TranslationContext context) {
  translateHelper(transform, context);
}
 

import org.apache.beam.sdk.transforms.windowing.Window; //导入方法依赖的package包/类
/**
 * Checks if the window transformation should be applied or skipped.
 *
 * <p>Avoid running assign windows if both source and destination are global window or if the user
 * has not specified the WindowFn (meaning they are just messing with triggering or allowed
 * lateness).
 *
 * @param transform The {@link Window.Assign} transformation.
 * @param context The {@link EvaluationContext}.
 * @param <T> PCollection type.
 * @param <W> {@link BoundedWindow} type.
 * @return if to apply the transformation.
 */
public static <T, W extends BoundedWindow> boolean skipAssignWindows(
    Window.Assign<T> transform, EvaluationContext context) {
  @SuppressWarnings("unchecked")
  WindowFn<? super T, W> windowFn = (WindowFn<? super T, W>) transform.getWindowFn();
  return windowFn == null
      || (context.getInput(transform).getWindowingStrategy().getWindowFn()
              instanceof GlobalWindows
          && windowFn instanceof GlobalWindows);
}