Java ParDo.SingleOutput方法代码示例

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
/**
 * Return a transform to pass-through events, but count them as they go by.
 */
public static ParDo.SingleOutput<Event, Event> snoop(final String name) {
  return ParDo.of(new DoFn<Event, Event>() {
    final Counter eventCounter = Metrics.counter(name, "events");
    final Counter newPersonCounter = Metrics.counter(name, "newPersons");
    final Counter newAuctionCounter = Metrics.counter(name, "newAuctions");
    final Counter bidCounter = Metrics.counter(name, "bids");
    final Counter endOfStreamCounter = Metrics.counter(name, "endOfStream");

    @ProcessElement
    public void processElement(ProcessContext c) {
      eventCounter.inc();
      if (c.element().newPerson != null) {
        newPersonCounter.inc();
      } else if (c.element().newAuction != null) {
        newAuctionCounter.inc();
      } else if (c.element().bid != null) {
        bidCounter.inc();
      } else {
        endOfStreamCounter.inc();
      }
      info("%s snooping element %s", name, c.element());
      c.output(c.element());
    }
  });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
/**
 * Return a transform to keep the CPU busy for given milliseconds on every record.
 */
public static <T> ParDo.SingleOutput<T, T> cpuDelay(String name, final long delayMs) {
  return ParDo.of(new DoFn<T, T>() {
                @ProcessElement
                public void processElement(ProcessContext c) {
                  long now = System.currentTimeMillis();
                  long end = now + delayMs;
                  while (now < end) {
                    // Find plaintext which hashes to HASH in lowest MASK bits.
                    // Values chosen to roughly take 1ms on typical workstation.
                    long p = INIT_PLAINTEXT;
                    while (true) {
                      long t = Hashing.murmur3_128().hashLong(p).asLong();
                      if ((t & MASK) == (HASH & MASK)) {
                        break;
                      }
                      p++;
                    }
                    now = System.currentTimeMillis();
                  }
                  c.output(c.element());
                }
              });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
/**
 * A {@link PTransformMatcher} that matches a {@link ParDo.SingleOutput} containing a {@link DoFn}
 * that is splittable, as signified by {@link ProcessElementMethod#isSplittable()}.
 */
public static PTransformMatcher splittableParDoSingle() {
  return new PTransformMatcher() {
    @Override
    public boolean matches(AppliedPTransform<?, ?, ?> application) {
      PTransform<?, ?> transform = application.getTransform();
      if (transform instanceof ParDo.SingleOutput) {
        DoFn<?, ?> fn = ((ParDo.SingleOutput<?, ?>) transform).getFn();
        DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn);
        return signature.processElement().isSplittable();
      }
      return false;
    }

    @Override
    public String toString() {
      return MoreObjects.toStringHelper("SplittableParDoSingleMatcher").toString();
    }
  };
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
/**
 * A {@link PTransformMatcher} that matches a {@link ParDo.SingleOutput} containing a {@link DoFn}
 * that uses state or timers, as specified by {@link DoFnSignature#usesState()} and {@link
 * DoFnSignature#usesTimers()}.
 */
public static PTransformMatcher stateOrTimerParDoSingle() {
  return new PTransformMatcher() {
    @Override
    public boolean matches(AppliedPTransform<?, ?, ?> application) {
      PTransform<?, ?> transform = application.getTransform();
      if (transform instanceof ParDo.SingleOutput) {
        DoFn<?, ?> fn = ((ParDo.SingleOutput<?, ?>) transform).getFn();
        DoFnSignature signature = DoFnSignatures.signatureForDoFn(fn);
        return signature.usesState() || signature.usesTimers();
      }
      return false;
    }

    @Override
    public String toString() {
      return MoreObjects.toStringHelper("StateOrTimerParDoSingleMatcher").toString();
    }
  };
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
/**
 * A {@link PTransformMatcher} which matches a {@link ParDo.SingleOutput} or {@link
 * ParDo.MultiOutput} where the {@link DoFn} is of the provided type.
 */
public static PTransformMatcher parDoWithFnType(final Class<? extends DoFn> fnType) {
  return new PTransformMatcher() {
    @Override
    public boolean matches(AppliedPTransform<?, ?, ?> application) {
      DoFn<?, ?> fn;
      if (application.getTransform() instanceof ParDo.SingleOutput) {
        fn = ((ParDo.SingleOutput) application.getTransform()).getFn();
      } else if (application.getTransform() instanceof ParDo.MultiOutput) {
        fn = ((ParDo.MultiOutput) application.getTransform()).getFn();
      } else {
        return false;
      }
      return fnType.equals(fn.getClass());
    }

    @Override
    public String toString() {
      return MoreObjects.toStringHelper("ParDoWithFnTypeMatcher")
          .add("fnType", fnType)
          .toString();
    }
  };
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
@Test
public void getReplacementTransformGetFn() {
  DoFn<Integer, Long> originalFn = new ToLongFn();
  ParDo.SingleOutput<Integer, Long> originalTransform = ParDo.of(originalFn);
  PCollection<? extends Integer> input = pipeline.apply(Create.of(1, 2, 3));
  AppliedPTransform<
          PCollection<? extends Integer>, PCollection<Long>, ParDo.SingleOutput<Integer, Long>>
      application =
          AppliedPTransform.of(
              "original",
              input.expand(),
              input.apply(originalTransform).expand(),
              originalTransform,
              pipeline);

  PTransformReplacement<PCollection<? extends Integer>, PCollection<Long>> replacementTransform =
      factory.getReplacementTransform(application);
  ParDoSingle<Integer, Long> parDoSingle =
      (ParDoSingle<Integer, Long>) replacementTransform.getTransform();

  assertThat(parDoSingle.getFn(), equalTo(originalTransform.getFn()));
  assertThat(parDoSingle.getFn(), equalTo(originalFn));
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
/**
 * Return a transform to count and discard each element.
 */
public static <T> ParDo.SingleOutput<T, Void> devNull(final String name) {
  return ParDo.of(new DoFn<T, Void>() {
    final Counter discardedCounterMetric = Metrics.counter(name, "discarded");

    @ProcessElement
    public void processElement(ProcessContext c) {
      discardedCounterMetric.inc();
    }
  });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
/**
 * Return a transform to log each element, passing it through unchanged.
 */
public static <T> ParDo.SingleOutput<T, T> log(final String name) {
  return ParDo.of(new DoFn<T, T>() {
    @ProcessElement
    public void processElement(ProcessContext c) {
      LOG.info("%s: %s", name, c.element());
      c.output(c.element());
    }
  });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
/**
 * Return a transform to format each element as a string.
 */
public static <T> ParDo.SingleOutput<T, String> format(final String name) {
  return ParDo.of(new DoFn<T, String>() {
    final Counter recordCounterMetric = Metrics.counter(name, "records");

    @ProcessElement
    public void processElement(ProcessContext c) {
      recordCounterMetric.inc();
      c.output(c.element().toString());
    }
  });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
/**
 * Return a transform to make explicit the timestamp of each element.
 */
public static <T> ParDo.SingleOutput<T, TimestampedValue<T>> stamp(String name) {
  return ParDo.of(new DoFn<T, TimestampedValue<T>>() {
    @ProcessElement
    public void processElement(ProcessContext c) {
      c.output(TimestampedValue.of(c.element(), c.timestamp()));
    }
  });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
/**
 * Return a transform to cast each element to {@link KnownSize}.
 */
private static <T extends KnownSize> ParDo.SingleOutput<T, KnownSize> castToKnownSize() {
  return ParDo.of(new DoFn<T, KnownSize>() {
                @ProcessElement
                public void processElement(ProcessContext c) {
                  c.output(c.element());
                }
              });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
public static ParDo.SingleOutput<Event, BeamRecord> parDo() {
  return ParDo.of(new DoFn<Event, BeamRecord>() {
    @ProcessElement
    public void processElement(ProcessContext c) {
      BeamRecord beamRecord = INSTANCE.toRecord(c.element());
      c.output(beamRecord);
    }
  });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
public static ParDo.SingleOutput<BeamRecord, BeamRecordSize> parDo() {
  return ParDo.of(new DoFn<BeamRecord, BeamRecordSize>() {
    @ProcessElement
    public void processElement(ProcessContext c) {
      c.output(BeamRecordSize.of(c.element()));
    }
  });
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
@Test
public void testFinishSpecifyingShouldFailIfNoCoderInferrable() {
  p.enableAbandonedNodeEnforcement(false);
  PCollection<Integer> created = p.apply(Create.of(1, 2, 3));
  ParDo.SingleOutput<Integer, EmptyClass> uninferrableParDo = ParDo.of(new EmptyClassDoFn());
  PCollection<EmptyClass> unencodable =
      created.apply(uninferrableParDo);

  thrown.expect(IllegalStateException.class);
  thrown.expectMessage("Unable to return a default Coder");
  thrown.expectMessage("Inferring a Coder from the CoderRegistry failed");

  unencodable.finishSpecifying(created, uninferrableParDo);
}
 

import org.apache.beam.sdk.transforms.ParDo; //导入方法依赖的package包/类
private ParDo.SingleOutput<String, String> printParDo(final String prefix) {
  return ParDo.of(new DoFn<String, String>() {

    @ProcessElement
    public void processElement(final ProcessContext c) {
      System.out.println(prefix + " " + c.element());
    }
  });
}