Java PCollection.isBounded方法代码示例

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
public PCollection<BeamRecord> sideInputJoin(
    PCollection<KV<BeamRecord, BeamRecord>> extractedLeftRows,
    PCollection<KV<BeamRecord, BeamRecord>> extractedRightRows,
    BeamRecord leftNullRow, BeamRecord rightNullRow) {
  // we always make the Unbounded table on the left to do the sideInput join
  // (will convert the result accordingly before return)
  boolean swapped = (extractedLeftRows.isBounded() == PCollection.IsBounded.BOUNDED);
  JoinRelType realJoinType =
      (swapped && joinType != JoinRelType.INNER) ? JoinRelType.LEFT : joinType;

  PCollection<KV<BeamRecord, BeamRecord>> realLeftRows =
      swapped ? extractedRightRows : extractedLeftRows;
  PCollection<KV<BeamRecord, BeamRecord>> realRightRows =
      swapped ? extractedLeftRows : extractedRightRows;
  BeamRecord realRightNullRow = swapped ? leftNullRow : rightNullRow;

  // swapped still need to pass down because, we need to swap the result back.
  return sideInputJoinHelper(realJoinType, realLeftRows, realRightRows,
      realRightNullRow, swapped);
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
public static void applicableTo(PCollection<?> input) {
  WindowingStrategy<?, ?> windowingStrategy = input.getWindowingStrategy();
  // Verify that the input PCollection is bounded, or that there is windowing/triggering being
  // used. Without this, the watermark (at end of global window) will never be reached.
  if (windowingStrategy.getWindowFn() instanceof GlobalWindows
      && windowingStrategy.getTrigger() instanceof DefaultTrigger
      && input.isBounded() != IsBounded.BOUNDED) {
    throw new IllegalStateException("GroupByKey cannot be applied to non-bounded PCollection in "
        + "the GlobalWindow without a trigger. Use a Window.into or Window.triggering transform "
        + "prior to GroupByKey.");
  }

  // Validate the window merge function.
  if (windowingStrategy.getWindowFn() instanceof InvalidWindows) {
    String cause = ((InvalidWindows<?>) windowingStrategy.getWindowFn()).getCause();
    throw new IllegalStateException(
        "GroupByKey must have a valid Window merge function.  "
            + "Invalid because: " + cause);
  }
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Override
public PDone expand(PCollection<T> input) {
  if (getTopicProvider() == null) {
    throw new IllegalStateException("need to set the topic of a PubsubIO.Write transform");
  }
  switch (input.isBounded()) {
    case BOUNDED:
      input.apply(ParDo.of(new PubsubBoundedWriter()));
      return PDone.in(input.getPipeline());
    case UNBOUNDED:
      return input.apply(MapElements.via(getFormatFn())).apply(new PubsubUnboundedSink(
          FACTORY,
          NestedValueProvider.of(getTopicProvider(), new TopicPathTranslator()),
          getTimestampAttribute(),
          getIdAttribute(),
          100 /* numShards */));
  }
  throw new RuntimeException(); // cases are exhaustive.
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
private Method resolveMethod(PCollection<T> input) {
  if (getMethod() != Method.DEFAULT) {
    return getMethod();
  }
  // By default, when writing an Unbounded PCollection, we use StreamingInserts and
  // BigQuery's streaming import API.
  return (input.isBounded() == IsBounded.UNBOUNDED)
      ? Method.STREAMING_INSERTS
      : Method.FILE_LOADS;
}
 

import org.apache.beam.sdk.values.PCollection; //导入方法依赖的package包/类
@Override
public WriteFilesResult<DestinationT> expand(PCollection<UserT> input) {
  if (input.isBounded() == IsBounded.UNBOUNDED) {
    checkArgument(
        getWindowedWrites(),
        "Must use windowed writes when applying %s to an unbounded PCollection",
        WriteFiles.class.getSimpleName());
    // The reason for this is https://issues.apache.org/jira/browse/BEAM-1438
    // and similar behavior in other runners.
    checkArgument(
        getComputeNumShards() != null || getNumShardsProvider() != null,
        "When applying %s to an unbounded PCollection, "
            + "must specify number of output shards explicitly",
        WriteFiles.class.getSimpleName());
  }
  this.writeOperation = getSink().createWriteOperation();
  this.writeOperation.setWindowedWrites(getWindowedWrites());

  if (!getWindowedWrites()) {
    // Re-window the data into the global window and remove any existing triggers.
    input =
        input.apply(
            "RewindowIntoGlobal",
            Window.<UserT>into(new GlobalWindows())
                .triggering(DefaultTrigger.of())
                .discardingFiredPanes());
  }

  Coder<DestinationT> destinationCoder;
  try {
    destinationCoder =
        getDynamicDestinations()
            .getDestinationCoderWithDefault(input.getPipeline().getCoderRegistry());
    destinationCoder.verifyDeterministic();
  } catch (CannotProvideCoderException | NonDeterministicException e) {
    throw new RuntimeException(e);
  }
  @SuppressWarnings("unchecked")
  Coder<BoundedWindow> windowCoder =
      (Coder<BoundedWindow>) input.getWindowingStrategy().getWindowFn().windowCoder();
  FileResultCoder<DestinationT> fileResultCoder =
      FileResultCoder.of(windowCoder, destinationCoder);

  PCollectionView<Integer> numShardsView =
      (getComputeNumShards() == null) ? null : input.apply(getComputeNumShards());

  PCollection<FileResult<DestinationT>> tempFileResults =
      (getComputeNumShards() == null && getNumShardsProvider() == null)
          ? input.apply(
              "WriteUnshardedBundlesToTempFiles",
              new WriteUnshardedBundlesToTempFiles(destinationCoder, fileResultCoder))
          : input.apply(
              "WriteShardedBundlesToTempFiles",
              new WriteShardedBundlesToTempFiles(
                  destinationCoder, fileResultCoder, numShardsView));

  return tempFileResults
      .apply("GatherTempFileResults", new GatherResults<>(fileResultCoder))
      .apply(
          "FinalizeTempFileBundles",
          new FinalizeTempFileBundles(numShardsView, destinationCoder));
}