Java ParDo.of方法代码示例

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
/**
 * Return a transform to pass-through events, but count them as they go by.
 */
public static ParDo.SingleOutput<Event, Event> snoop(final String name) {
  return ParDo.of(new DoFn<Event, Event>() {
    final Counter eventCounter = Metrics.counter(name, "events");
    final Counter newPersonCounter = Metrics.counter(name, "newPersons");
    final Counter newAuctionCounter = Metrics.counter(name, "newAuctions");
    final Counter bidCounter = Metrics.counter(name, "bids");
    final Counter endOfStreamCounter = Metrics.counter(name, "endOfStream");

    @ProcessElement
    public void processElement(ProcessContext c) {
      eventCounter.inc();
      if (c.element().newPerson != null) {
        newPersonCounter.inc();
      } else if (c.element().newAuction != null) {
        newAuctionCounter.inc();
      } else if (c.element().bid != null) {
        bidCounter.inc();
      } else {
        endOfStreamCounter.inc();
      }
      info("%s snooping element %s", name, c.element());
      c.output(c.element());
    }
  });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
/**
 * Return a transform to keep the CPU busy for given milliseconds on every record.
 */
public static <T> ParDo.SingleOutput<T, T> cpuDelay(String name, final long delayMs) {
  return ParDo.of(new DoFn<T, T>() {
                @ProcessElement
                public void processElement(ProcessContext c) {
                  long now = System.currentTimeMillis();
                  long end = now + delayMs;
                  while (now < end) {
                    // Find plaintext which hashes to HASH in lowest MASK bits.
                    // Values chosen to roughly take 1ms on typical workstation.
                    long p = INIT_PLAINTEXT;
                    while (true) {
                      long t = Hashing.murmur3_128().hashLong(p).asLong();
                      if ((t & MASK) == (HASH & MASK)) {
                        break;
                      }
                      p++;
                    }
                    now = System.currentTimeMillis();
                  }
                  c.output(c.element());
                }
              });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
private PTransform<? super PCollection<BeamRecord>, PCollection<BeamRecord>> logBytesMetric(
    final BeamRecordCoder coder) {

  return ParDo.of(new DoFn<BeamRecord, BeamRecord>() {
    private final Counter bytesMetric = Metrics.counter(name , "bytes");

    @ProcessElement
    public void processElement(ProcessContext c) throws CoderException, IOException {
      ByteArrayOutputStream outStream = new ByteArrayOutputStream();
      coder.encode(c.element(), outStream, Coder.Context.OUTER);
      byte[] byteArray = outStream.toByteArray();
      bytesMetric.inc((long) byteArray.length);
      ByteArrayInputStream inStream = new ByteArrayInputStream(byteArray);
      BeamRecord record = coder.decode(inStream, Coder.Context.OUTER);
      c.output(record);
    }
  });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
@Test
public void testPrimitiveTransform() {
  PTransform<? super PCollection<Integer>, ? super PCollection<Integer>> myTransform = ParDo.of(
      new DoFn<Integer, Integer>() {
    @ProcessElement
    public void processElement(ProcessContext c) throws Exception {}

    @Override
    public void populateDisplayData(DisplayData.Builder builder) {
      builder.add(DisplayData.item("foo", "bar"));
    }
  });

  DisplayDataEvaluator evaluator = DisplayDataEvaluator.create();
  Set<DisplayData> displayData = evaluator.displayDataForPrimitiveTransforms(myTransform);

  assertThat(displayData, hasItem(hasDisplayItem("foo")));
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
@Test
public void getReplacementTransformGetFn() {
  DoFn<Integer, Long> originalFn = new ToLongFn();
  ParDo.SingleOutput<Integer, Long> originalTransform = ParDo.of(originalFn);
  PCollection<? extends Integer> input = pipeline.apply(Create.of(1, 2, 3));
  AppliedPTransform<
          PCollection<? extends Integer>, PCollection<Long>, ParDo.SingleOutput<Integer, Long>>
      application =
          AppliedPTransform.of(
              "original",
              input.expand(),
              input.apply(originalTransform).expand(),
              originalTransform,
              pipeline);

  PTransformReplacement<PCollection<? extends Integer>, PCollection<Long>> replacementTransform =
      factory.getReplacementTransform(application);
  ParDoSingle<Integer, Long> parDoSingle =
      (ParDoSingle<Integer, Long>) replacementTransform.getTransform();

  assertThat(parDoSingle.getFn(), equalTo(originalTransform.getFn()));
  assertThat(parDoSingle.getFn(), equalTo(originalFn));
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
/**
 * Return a transform to count and discard each element.
 */
public static <T> ParDo.SingleOutput<T, Void> devNull(final String name) {
  return ParDo.of(new DoFn<T, Void>() {
    final Counter discardedCounterMetric = Metrics.counter(name, "discarded");

    @ProcessElement
    public void processElement(ProcessContext c) {
      discardedCounterMetric.inc();
    }
  });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
/**
 * Return a transform to log each element, passing it through unchanged.
 */
public static <T> ParDo.SingleOutput<T, T> log(final String name) {
  return ParDo.of(new DoFn<T, T>() {
    @ProcessElement
    public void processElement(ProcessContext c) {
      LOG.info("%s: %s", name, c.element());
      c.output(c.element());
    }
  });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
/**
 * Return a transform to format each element as a string.
 */
public static <T> ParDo.SingleOutput<T, String> format(final String name) {
  return ParDo.of(new DoFn<T, String>() {
    final Counter recordCounterMetric = Metrics.counter(name, "records");

    @ProcessElement
    public void processElement(ProcessContext c) {
      recordCounterMetric.inc();
      c.output(c.element().toString());
    }
  });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
/**
 * Return a transform to make explicit the timestamp of each element.
 */
public static <T> ParDo.SingleOutput<T, TimestampedValue<T>> stamp(String name) {
  return ParDo.of(new DoFn<T, TimestampedValue<T>>() {
    @ProcessElement
    public void processElement(ProcessContext c) {
      c.output(TimestampedValue.of(c.element(), c.timestamp()));
    }
  });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
/**
 * Return a transform to cast each element to {@link KnownSize}.
 */
private static <T extends KnownSize> ParDo.SingleOutput<T, KnownSize> castToKnownSize() {
  return ParDo.of(new DoFn<T, KnownSize>() {
                @ProcessElement
                public void processElement(ProcessContext c) {
                  c.output(c.element());
                }
              });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
public static ParDo.SingleOutput<Event, BeamRecord> parDo() {
  return ParDo.of(new DoFn<Event, BeamRecord>() {
    @ProcessElement
    public void processElement(ProcessContext c) {
      BeamRecord beamRecord = INSTANCE.toRecord(c.element());
      c.output(beamRecord);
    }
  });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
public static ParDo.SingleOutput<BeamRecord, BeamRecordSize> parDo() {
  return ParDo.of(new DoFn<BeamRecord, BeamRecordSize>() {
    @ProcessElement
    public void processElement(ProcessContext c) {
      c.output(BeamRecordSize.of(c.element()));
    }
  });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
@Test
public void testFinishSpecifyingShouldFailIfNoCoderInferrable() {
  p.enableAbandonedNodeEnforcement(false);
  PCollection<Integer> created = p.apply(Create.of(1, 2, 3));
  ParDo.SingleOutput<Integer, EmptyClass> uninferrableParDo = ParDo.of(new EmptyClassDoFn());
  PCollection<EmptyClass> unencodable =
      created.apply(uninferrableParDo);

  thrown.expect(IllegalStateException.class);
  thrown.expectMessage("Unable to return a default Coder");
  thrown.expectMessage("Inferring a Coder from the CoderRegistry failed");

  unencodable.finishSpecifying(created, uninferrableParDo);
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
private ParDo.SingleOutput<String, String> printParDo(final String prefix) {
  return ParDo.of(new DoFn<String, String>() {

    @ProcessElement
    public void processElement(final ProcessContext c) {
      System.out.println(prefix + " " + c.element());
    }
  });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
/**
 * A test that demonstrates that the replacement transform has the Display Data of the
 * {@link ParDo.SingleOutput} it replaces.
 */
@Test
public void getReplacementTransformPopulateDisplayData() {
  ParDo.SingleOutput<Integer, Long> originalTransform = ParDo.of(new ToLongFn());
  DisplayData originalDisplayData = DisplayData.from(originalTransform);
  PCollection<? extends Integer> input = pipeline.apply(Create.of(1, 2, 3));
  AppliedPTransform<
      PCollection<? extends Integer>, PCollection<Long>, ParDo.SingleOutput<Integer, Long>>
      application =
      AppliedPTransform.of(
          "original",
          input.expand(),
          input.apply(originalTransform).expand(),
          originalTransform,
          pipeline);

  PTransformReplacement<PCollection<? extends Integer>, PCollection<Long>> replacement =
      factory.getReplacementTransform(application);
  DisplayData replacementDisplayData = DisplayData.from(replacement.getTransform());

  assertThat(replacementDisplayData, equalTo(originalDisplayData));

  DisplayData primitiveDisplayData =
      Iterables.getOnlyElement(
          DisplayDataEvaluator.create()
              .displayDataForPrimitiveTransforms(replacement.getTransform(), VarIntCoder.of()));
  assertThat(primitiveDisplayData, equalTo(replacementDisplayData));
}