Java PCollection.getCoder方法代码示例

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
private PCollection<Event> applyTyped(PCollection<Event> events) {
  final Coder<Event> coder = events.getCoder();
  return events
      // Force round trip through coder.
      .apply(name + ".Serialize",
          ParDo.of(new DoFn<Event, Event>() {
                private final Counter bytesMetric =
                  Metrics.counter(name , "bytes");

                @ProcessElement
                public void processElement(ProcessContext c) throws CoderException, IOException {
                  ByteArrayOutputStream outStream = new ByteArrayOutputStream();
                  coder.encode(c.element(), outStream, Coder.Context.OUTER);
                  byte[] byteArray = outStream.toByteArray();
                  bytesMetric.inc((long) byteArray.length);
                  ByteArrayInputStream inStream = new ByteArrayInputStream(byteArray);
                  Event event = coder.decode(inStream, Coder.Context.OUTER);
                  c.output(event);
                }
              }));
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
/**
 * Utility to get mapping between TupleTag and a coder.
 * @param outputs - A map of tuple tags and pcollections
 * @return mapping between TupleTag and a coder
 */
public static Map<TupleTag<?>, Coder<WindowedValue<?>>> getTupleTagCoders(
    Map<TupleTag<?>, PValue> outputs) {
  Map<TupleTag<?>, Coder<WindowedValue<?>>> coderMap = new HashMap<>(outputs.size());

  for (Map.Entry<TupleTag<?>, PValue> output : outputs.entrySet()) {
    // we get the first PValue as all of them are fro the same type.
    PCollection<?> pCollection = (PCollection<?>) output.getValue();
    Coder<?> coder = pCollection.getCoder();
    Coder<? extends BoundedWindow> wCoder =
        pCollection.getWindowingStrategy().getWindowFn().windowCoder();
    @SuppressWarnings("unchecked")
    Coder<WindowedValue<?>> windowedValueCoder =
        (Coder<WindowedValue<?>>) (Coder<?>) WindowedValue.getFullCoder(coder, wCoder);
    coderMap.put(output.getKey(), windowedValueCoder);
  }
  return coderMap;
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Override
public PCollection<KV<K, TimestampedValue<V>>> expand(PCollection<KV<K, V>> input) {
  KvCoder<K, V> coder = (KvCoder<K, V>) input.getCoder();
  return input
      .apply(
          ParDo.of(
              new DoFn<KV<K, V>, KV<K, TimestampedValue<V>>>() {
                @ProcessElement
                public void processElement(ProcessContext context) {
                  context.output(
                      KV.of(
                          context.element().getKey(),
                          TimestampedValue.of(
                              context.element().getValue(), context.timestamp())));
                }
              }))
      .setCoder(
          KvCoder.of(coder.getKeyCoder(), TimestampedValueCoder.of(coder.getValueCoder())));
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
private static <K, InputT, AccumT> Coder<AccumT> extractAccumulatorCoder(
    GlobalCombineFn<InputT, AccumT, ?> combineFn,
    AppliedPTransform<PCollection<KV<K, InputT>>, ?, Combine.PerKey<K, InputT, ?>> transform)
    throws CannotProvideCoderException {
  @SuppressWarnings("unchecked")
  PCollection<KV<K, InputT>> mainInput =
      (PCollection<KV<K, InputT>>)
          Iterables.getOnlyElement(TransformInputs.nonAdditionalInputs(transform));
  KvCoder<K, InputT> inputCoder = (KvCoder<K, InputT>) mainInput.getCoder();
  return AppliedCombineFn.withInputCoder(
          combineFn,
          transform.getPipeline().getCoderRegistry(),
          inputCoder,
          transform.getTransform().getSideInputs(),
          ((PCollection<?>) Iterables.getOnlyElement(transform.getOutputs().values()))
              .getWindowingStrategy())
      .getAccumulatorCoder();
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Override
public PCollection<KV<K, V>> expand(PCollection<KV<K, V>> input) {
  checkNotNull(input);
  checkArgument(input.getCoder() instanceof KvCoder,
      "Input specifiedCoder must be an instance of KvCoder, but was %s", input.getCoder());

  @SuppressWarnings("unchecked")
  KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();
  return input
      .apply("Reify Timestamps", ParDo.of(
        new DoFn<KV<K, V>, KV<K, TimestampedValue<V>>>() {
          @ProcessElement
          public void processElement(ProcessContext c) {
            c.output(KV.of(c.element().getKey(), TimestampedValue.of(c.element().getValue(),
                c.timestamp())));
          }
        })).setCoder(KvCoder.of(
            inputCoder.getKeyCoder(),
            TimestampedValue.TimestampedValueCoder.of(inputCoder.getValueCoder())))
      .apply("Latest Value", Combine.<K, TimestampedValue<V>, V>perKey(new LatestFn<V>()))
        .setCoder(inputCoder);
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Override
public PCollection<KV<Integer, Iterable<KV<KV<K, W>, WindowedValue<V>>>>>
expand(PCollection<KV<K, V>> input) {

  @SuppressWarnings("unchecked")
  Coder<W> windowCoder = (Coder<W>)
      input.getWindowingStrategy().getWindowFn().windowCoder();
  @SuppressWarnings("unchecked")
  KvCoder<K, V> inputCoder = (KvCoder<K, V>) input.getCoder();

  PCollection<KV<Integer, KV<KV<K, W>, WindowedValue<V>>>> keyedByHash;
  keyedByHash = input.apply(
      ParDo.of(new GroupByKeyHashAndSortByKeyAndWindowDoFn<K, V, W>(coder)));
  keyedByHash.setCoder(
      KvCoder.of(
          VarIntCoder.of(),
          KvCoder.of(KvCoder.of(inputCoder.getKeyCoder(), windowCoder),
              FullWindowedValueCoder.of(inputCoder.getValueCoder(), windowCoder))));

  return keyedByHash.apply(
      new GroupByKeyAndSortValuesOnly<Integer, KV<K, W>, WindowedValue<V>>());
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
private static void validateStateApplicableForInput(
    DoFn<?, ?> fn,
    PCollection<?> input) {
  Coder<?> inputCoder = input.getCoder();
  checkArgument(
      inputCoder instanceof KvCoder,
      "%s requires its input to use %s in order to use state and timers.",
      ParDo.class.getSimpleName(),
      KvCoder.class.getSimpleName());

  KvCoder<?, ?> kvCoder = (KvCoder<?, ?>) inputCoder;
  try {
      kvCoder.getKeyCoder().verifyDeterministic();
  } catch (Coder.NonDeterministicException exc) {
    throw new IllegalArgumentException(
        String.format(
            "%s requires a deterministic key coder in order to use state and timers",
            ParDo.class.getSimpleName()));
  }
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Override
public PCollection<KV<K, Long>> expand(PCollection<KV<K, V>> input) {
  KvCoder<K, V> inputCoder = (KvCoder<K, V>) input.getCoder();
  return input
      .apply(
          Combine.<K, V, HyperLogLogPlus>perKey(
              ApproximateDistinctFn.<V>create(inputCoder.getValueCoder())
                  .withPrecision(this.precision())
                  .withSparseRepresentation(this.sparsePrecision())))
      .apply("Retrieve Cardinality", ParDo.of(RetrieveCardinality.<K>perKey()));
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
/** Transforms the input {@link PCollection} into a singleton {@link Map} per window. */
private <W extends BoundedWindow> PCollection<?>
applyForSingletonFallback(PCollection<KV<K, V>> input) {
  @SuppressWarnings("unchecked")
  Coder<W> windowCoder = (Coder<W>)
      input.getWindowingStrategy().getWindowFn().windowCoder();

  @SuppressWarnings({"rawtypes", "unchecked"})
  KvCoder<K, V> inputCoder = (KvCoder) input.getCoder();

  @SuppressWarnings({"unchecked", "rawtypes"})
  Coder<Function<WindowedValue<V>, V>> transformCoder =
      (Coder) SerializableCoder.of(WindowedValueToValue.class);

  Coder<TransformedMap<K, WindowedValue<V>, V>> finalValueCoder =
      TransformedMapCoder.of(
          transformCoder,
          MapCoder.of(
              inputCoder.getKeyCoder(),
              FullWindowedValueCoder.of(inputCoder.getValueCoder(), windowCoder)));

  return BatchViewAsSingleton.<KV<K, V>, TransformedMap<K, WindowedValue<V>, V>,
      Map<K, V>,
      W> applyForSingleton(
      runner,
      input,
      new ToMapDoFn<K, V, W>(windowCoder),
      finalValueCoder,
      view);
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Override
public PCollection<Long> expand(PCollection<T> input) {
  Coder<T> coder = input.getCoder();
  return input.apply(
      Combine.globally(
          new ApproximateUniqueCombineFn<>(sampleSize, coder)));
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Override
public PCollection<KV<K, Iterable<InputT>>> expand(PCollection<KV<K, InputT>> input) {
  Duration allowedLateness = input.getWindowingStrategy().getAllowedLateness();

  checkArgument(
      input.getCoder() instanceof KvCoder,
      "coder specified in the input PCollection is not a KvCoder");
  KvCoder inputCoder = (KvCoder) input.getCoder();
  Coder<K> keyCoder = (Coder<K>) inputCoder.getCoderArguments().get(0);
  Coder<InputT> valueCoder = (Coder<InputT>) inputCoder.getCoderArguments().get(1);

  return input.apply(
      ParDo.of(new GroupIntoBatchesDoFn<>(batchSize, allowedLateness, keyCoder, valueCoder)));
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
/**
 * Returns the value coder for the given PCollection.  Assumes that the value
 * coder is an instance of {@code KvCoder<K, V>}.
 */
private <V> Coder<V> getValueCoder(PCollection<KV<K, V>> pCollection) {
  // Assumes that the PCollection uses a KvCoder.
  Coder<?> entryCoder = pCollection.getCoder();
  if (!(entryCoder instanceof KvCoder<?, ?>)) {
    throw new IllegalArgumentException("PCollection does not use a KvCoder");
  }
  @SuppressWarnings("unchecked")
  KvCoder<K, V> coder = (KvCoder<K, V>) entryCoder;
  return coder.getValueCoder();
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Test
public void testGloballyOutputCoder() {
  p.enableAbandonedNodeEnforcement(false);

  BigEndianLongCoder inputCoder = BigEndianLongCoder.of();

  PCollection<Long> output =
      p.apply(Create.of(1L, 2L).withCoder(inputCoder))
          .apply(Latest.<Long>globally());

  Coder<Long> outputCoder = output.getCoder();
  assertThat(outputCoder, instanceOf(NullableCoder.class));
  assertEquals(inputCoder, ((NullableCoder<?>) outputCoder).getValueCoder());
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Override
public PCollection<KV<K, OutputT>> expand(PCollection<KV<K, InputT>> input) {
  checkArgument(
      input.getCoder() instanceof KvCoder,
      "Expected input to have a %s of type %s, got %s",
      Coder.class.getSimpleName(),
      KvCoder.class.getSimpleName(),
      input.getCoder());
  KvCoder<K, InputT> inputCoder = (KvCoder<K, InputT>) input.getCoder();
  Coder<InputT> inputValueCoder = inputCoder.getValueCoder();
  Coder<AccumT> accumulatorCoder;
  try {
    accumulatorCoder =
        combineFn.getAccumulatorCoder(input.getPipeline().getCoderRegistry(), inputValueCoder);
  } catch (CannotProvideCoderException e) {
    throw new IllegalStateException(
        String.format(
            "Could not construct an Accumulator Coder with the provided %s %s",
            CombineFn.class.getSimpleName(), combineFn),
        e);
  }
  return input
      .apply(
          ParDo.of(
              new CombineInputs<>(
                  combineFn,
                  input.getWindowingStrategy().getTimestampCombiner(),
                  inputCoder.getKeyCoder())))
      .setCoder(KvCoder.of(inputCoder.getKeyCoder(), accumulatorCoder))
      .apply(GroupByKey.<K, AccumT>create())
      .apply(new MergeAndExtractAccumulatorOutput<>(combineFn, outputCoder));
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
public <T> Coder<WindowedValue<T>> getCoder(PCollection<T> collection) {
  Coder<T> valueCoder = collection.getCoder();

  return WindowedValue.getFullCoder(
      valueCoder,
      collection.getWindowingStrategy().getWindowFn().windowCoder());
}