Java PCollection.createPrimitiveOutputInternal方法代码示例

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Override
public PCollection<KV<K, Iterable<V>>> expand(PCollection<KV<K, V>> input) {
  applicableTo(input);

  // Verify that the input Coder<KV<K, V>> is a KvCoder<K, V>, and that
  // the key coder is deterministic.
  Coder<K> keyCoder = getKeyCoder(input.getCoder());
  try {
    keyCoder.verifyDeterministic();
  } catch (NonDeterministicException e) {
    throw new IllegalStateException(
        "the keyCoder of a GroupByKey must be deterministic", e);
  }

  // This primitive operation groups by the combination of key and window,
  // merging windows as needed, using the windows assigned to the
  // key/value input elements and the window merge operation of the
  // window function associated with the input PCollection.
  return PCollection.createPrimitiveOutputInternal(
      input.getPipeline(),
      updateWindowingStrategy(input.getWindowingStrategy()),
      input.isBounded(),
      getOutputKvCoder(input.getCoder()));
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Test
public void getDefaultOutputCoderDelegates() throws Exception {
  @SuppressWarnings("unchecked")
  PCollection<Integer> input =
      PCollection.createPrimitiveOutputInternal(
          null /* pipeline */,
          WindowingStrategy.globalDefault(),
          PCollection.IsBounded.BOUNDED,
          null /* coder */);
  @SuppressWarnings("unchecked")
  PCollection<String> output = PCollection.createPrimitiveOutputInternal(
      null /* pipeline */,
      WindowingStrategy.globalDefault(),
      PCollection.IsBounded.BOUNDED,
      null /* coder */);
  @SuppressWarnings("unchecked")
  Coder<String> outputCoder = Mockito.mock(Coder.class);

  Mockito.when(delegate.expand(input)).thenReturn(output);
  Mockito.when(delegate.getDefaultOutputCoder(input, output)).thenReturn(outputCoder);
  assertThat(forwarding.expand(input).getCoder(), equalTo(outputCoder));
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Test
public void flattenWithDuplicateInputsWithDuplicates() {
  PCollection<Integer> duplicate =
      PCollection.createPrimitiveOutputInternal(
          p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarIntCoder.of());
  AppliedPTransform application =
      AppliedPTransform
          .<PCollectionList<Integer>, PCollection<Integer>, Flatten.PCollections<Integer>>of(
              "Flatten",
              ImmutableMap.<TupleTag<?>, PValue>builder()
                  .put(new TupleTag<Integer>(), duplicate)
                  .put(new TupleTag<Integer>(), duplicate)
                  .build(),
              Collections.<TupleTag<?>, PValue>singletonMap(
                  new TupleTag<Integer>(),
                  PCollection.createPrimitiveOutputInternal(
                      p, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarIntCoder.of())),
              Flatten.<Integer>pCollections(),
              p);

  assertThat(PTransformMatchers.flattenWithDuplicateInputs().matches(application), is(true));
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Test
public void classEqualToDoesNotMatchSubclass() {
  class MyPTransform extends PTransform<PCollection<KV<String, Integer>>, PCollection<Integer>> {
    @Override
    public PCollection<Integer> expand(PCollection<KV<String, Integer>> input) {
      return PCollection.createPrimitiveOutputInternal(
          input.getPipeline(), input.getWindowingStrategy(), input.isBounded(), VarIntCoder.of());
    }
  }
  PTransformMatcher matcher = PTransformMatchers.classEqualTo(MyPTransform.class);
  MyPTransform subclass = new MyPTransform() {};

  assertThat(subclass.getClass(), not(Matchers.<Class<?>>equalTo(MyPTransform.class)));
  assertThat(subclass, instanceOf(MyPTransform.class));

  AppliedPTransform<?, ?, ?> application =
      getAppliedTransform(subclass);

  assertThat(matcher.matches(application), is(false));
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Override
public PCollection<T> expand(PCollectionList<T> inputs) {
  WindowingStrategy<?, ?> windowingStrategy;
  IsBounded isBounded = IsBounded.BOUNDED;
  if (!inputs.getAll().isEmpty()) {
    windowingStrategy = inputs.get(0).getWindowingStrategy();
    for (PCollection<?> input : inputs.getAll()) {
      WindowingStrategy<?, ?> other = input.getWindowingStrategy();
      if (!windowingStrategy.getWindowFn().isCompatible(other.getWindowFn())) {
        throw new IllegalStateException(
            "Inputs to Flatten had incompatible window windowFns: "
            + windowingStrategy.getWindowFn() + ", " + other.getWindowFn());
      }

      if (!windowingStrategy.getTrigger().isCompatible(other.getTrigger())) {
        throw new IllegalStateException(
            "Inputs to Flatten had incompatible triggers: "
            + windowingStrategy.getTrigger() + ", " + other.getTrigger());
      }
      isBounded = isBounded.and(input.isBounded());
    }
  } else {
    windowingStrategy = WindowingStrategy.globalDefault();
  }

  return PCollection.createPrimitiveOutputInternal(
      inputs.getPipeline(),
      windowingStrategy,
      isBounded,
      // Take coder from first collection. If there are none, will be left unspecified.
      inputs.getAll().isEmpty() ? null : inputs.get(0).getCoder());
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@SuppressWarnings({"rawtypes", "unchecked"})
@Override
public PCollection<KV<K, Iterable<WindowedValue<V>>>> expand(PCollection<KV<K, V>> input) {
  return PCollection.createPrimitiveOutputInternal(
      input.getPipeline(),
      input.getWindowingStrategy(),
      input.isBounded(),
      (Coder) GroupByKey.getOutputKvCoder(input.getCoder()));
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Override
public PCollection<KeyedWorkItem<KeyT, InputT>> expand(PCollection<KV<KeyT, InputT>> input) {
  KvCoder<KeyT, InputT> kvCoder = (KvCoder<KeyT, InputT>) input.getCoder();
  return PCollection.createPrimitiveOutputInternal(
      input.getPipeline(),
      WindowingStrategy.globalDefault(),
      input.isBounded(),
      KeyedWorkItemCoder.of(
          kvCoder.getKeyCoder(),
          kvCoder.getValueCoder(),
          input.getWindowingStrategy().getWindowFn().windowCoder()));
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Test
public void emptyCompositeSucceeds() {
  PCollection<Long> created =
      PCollection.createPrimitiveOutputInternal(
          pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarLongCoder.of());
  TransformHierarchy.Node node = hierarchy.pushNode("Create", PBegin.in(pipeline), Create.of(1));
  hierarchy.setOutput(created);
  hierarchy.popNode();
  PCollectionList<Long> pcList = PCollectionList.of(created);

  TransformHierarchy.Node emptyTransform =
      hierarchy.pushNode(
          "Extract",
          pcList,
          new PTransform<PCollectionList<Long>, PCollection<Long>>() {
            @Override
            public PCollection<Long> expand(PCollectionList<Long> input) {
              return input.get(0);
            }
          });
  hierarchy.setOutput(created);
  hierarchy.popNode();
  assertThat(hierarchy.getProducer(created), equalTo(node));
  assertThat(
      "A Transform that produces non-primitive output should be composite",
      emptyTransform.isCompositeNode(),
      is(true));
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Test
public void producingOwnAndOthersOutputsFails() {
  PCollection<Long> created =
      PCollection.createPrimitiveOutputInternal(
          pipeline, WindowingStrategy.globalDefault(), IsBounded.BOUNDED, VarLongCoder.of());
  hierarchy.pushNode("Create", PBegin.in(pipeline), Create.of(1));
  hierarchy.setOutput(created);
  hierarchy.popNode();
  PCollectionList<Long> pcList = PCollectionList.of(created);

  final PCollectionList<Long> appended =
      pcList.and(
          PCollection.createPrimitiveOutputInternal(
                  pipeline,
                  WindowingStrategy.globalDefault(),
                  IsBounded.BOUNDED,
                  VarLongCoder.of())
              .setName("prim"));
  hierarchy.pushNode(
      "AddPc",
      pcList,
      new PTransform<PCollectionList<Long>, PCollectionList<Long>>() {
        @Override
        public PCollectionList<Long> expand(PCollectionList<Long> input) {
          return appended;
        }
      });
  thrown.expect(IllegalArgumentException.class);
  thrown.expectMessage("contains a primitive POutput produced by it");
  thrown.expectMessage("AddPc");
  thrown.expectMessage("Create");
  thrown.expectMessage(appended.expand().toString());
  hierarchy.setOutput(appended);
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Override
public PCollection<T> expand(PBegin input) {
  runner.setClockSupplier(new TestClockSupplier());
  return PCollection.createPrimitiveOutputInternal(
      input.getPipeline(),
      WindowingStrategy.globalDefault(),
      IsBounded.UNBOUNDED,
      original.getValueCoder());
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
private static PCollection<?> unionSideInputs(
    List<PCollectionView<?>> sideInputs, TranslationContext context) {
  checkArgument(sideInputs.size() > 1, "requires multiple side inputs");
  // flatten and assign union tag
  List<PCollection<Object>> sourceCollections = new ArrayList<>();
  Map<PCollection<?>, Integer> unionTags = new HashMap<>();
  PCollection<Object> firstSideInput = context.getViewInput(sideInputs.get(0));
  for (int i = 0; i < sideInputs.size(); i++) {
    PCollectionView<?> sideInput = sideInputs.get(i);
    PCollection<?> sideInputCollection = context.getViewInput(sideInput);
    if (!sideInputCollection
        .getWindowingStrategy()
        .equals(firstSideInput.getWindowingStrategy())) {
      // TODO: check how to handle this in stream codec
      //String msg = "Multiple side inputs with different window strategies.";
      //throw new UnsupportedOperationException(msg);
      LOG.warn(
          "Side inputs union with different windowing strategies {} {}",
          firstSideInput.getWindowingStrategy(),
          sideInputCollection.getWindowingStrategy());
    }
    if (!sideInputCollection.getCoder().equals(firstSideInput.getCoder())) {
      String msg = context.getFullName() + ": Multiple side inputs with different coders.";
      throw new UnsupportedOperationException(msg);
    }
    sourceCollections.add(context.<PCollection<Object>>getViewInput(sideInput));
    unionTags.put(sideInputCollection, i);
  }

  PCollection<Object> resultCollection =
      PCollection.createPrimitiveOutputInternal(
          firstSideInput.getPipeline(),
          firstSideInput.getWindowingStrategy(),
          firstSideInput.isBounded(),
          firstSideInput.getCoder());
  FlattenPCollectionTranslator.flattenCollections(
      sourceCollections, unionTags, resultCollection, context);
  return resultCollection;
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Override
public PCollection<KV<K, Iterable<V>>> expand(PCollection<KeyedWorkItem<K, V>> input) {
  KeyedWorkItemCoder<K, V> inputCoder = getKeyedWorkItemCoder(input.getCoder());
  return PCollection.createPrimitiveOutputInternal(
      input.getPipeline(), outputWindowingStrategy, input.isBounded(),
      KvCoder.of(inputCoder.getKeyCoder(), IterableCoder.of(inputCoder.getElementCoder())));
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
public static PCollection<?> fromProto(
    RunnerApi.PCollection pCollection, Pipeline pipeline, RehydratedComponents components)
    throws IOException {

  Coder<?> coder = components.getCoder(pCollection.getCoderId());
  return PCollection.createPrimitiveOutputInternal(
      pipeline,
      components.getWindowingStrategy(pCollection.getWindowingStrategyId()),
      fromProto(pCollection.getIsBounded()),
      (Coder) coder);
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Override
public PCollection<KV<K, V>> apply(PInput input) {
  // Spark streaming micro batches are bounded by default
  return PCollection.createPrimitiveOutputInternal(input.getPipeline(),
      WindowingStrategy.globalDefault(), PCollection.IsBounded.UNBOUNDED);
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Override
public PCollection<T> expand(PCollection<T> input) {
  return PCollection.createPrimitiveOutputInternal(
      input.getPipeline(), updatedStrategy, input.isBounded(), input.getCoder());
}