Java DoFn类代码示例

import org.apache.crunch.DoFn; //导入依赖的package包/类
@Test
public void testDetach() {
  Collection<TestAvroRecord> expected = Lists.newArrayList(
          new TestAvroRecord(new Utf8("something"), new Utf8("*"), 1L),
          new TestAvroRecord(new Utf8("something"), new Utf8("**"), 1L),
          new TestAvroRecord(new Utf8("something"), new Utf8("***"), 1L)
  );
  DoFn<Pair<String, Iterable<TestAvroRecord>>, Collection<TestAvroRecord>> doFn =
          DoFns.detach(new CollectingMapFn(), Avros.specifics(TestAvroRecord.class));
  Pair<String, Iterable<TestAvroRecord>> input = Pair.of("key", (Iterable<TestAvroRecord>) new AvroIterable());
  InMemoryEmitter<Collection<TestAvroRecord>> emitter = new InMemoryEmitter<Collection<TestAvroRecord>>();

  doFn.configure(new Configuration());
  doFn.initialize();
  doFn.process(input, emitter);
  doFn.cleanup(emitter);

  assertEquals(expected, emitter.getOutput().get(0));
}
 

import org.apache.crunch.DoFn; //导入依赖的package包/类
public DetachingDoFn(DoFn<Pair<K, Iterable<V>>, T> reduceFn, PType<V> valueType) {
  this.reduceFn = reduceFn;
  this.valueType = valueType;
}
 

import org.apache.crunch.DoFn; //导入依赖的package包/类
/**
 * "Reduce" DoFn wrapper which detaches the values in the iterable, preventing the unexpected behaviour related to
 * object reuse often observed when using Avro. Wrap your DoFn in a detach(...) and pass in a PType for the Iterable
 * value, and then you'll be handed an Iterable of real distinct objects, instead of the same object being handed to
 * you multiple times with different data.
 *
 * You should use this when you have a parallelDo after a groupBy, and you'd like to capture the objects arriving in
 * the Iterable part of the incoming Pair and pass it through to the output (for example if you want to create an
 * array of outputs from the values to be output as one record).
 *
 * The will incur a performance hit, as it means that every object read from the Iterable will allocate a new Java
 * object for the record and objects for all its non-primitive fields too. If you are rolling up records into a
 * collection then this will be necessary anyway, but if you are only outputting derived data this may impact the
 * speed and memory usage of your job unnecessarily.
 *
 * @param reduceFn Underlying DoFn to wrap
 * @param valueType PType of the object contained within the Iterable
 * @param <K> Reduce key
 * @param <V> Iterable value
 * @param <T> Output type of DoFn
 * @return DoFn which will detach values for you
 */
public static <K, V, T> DoFn<Pair<K, Iterable<V>>, T> detach(final DoFn<Pair<K, Iterable<V>>, T> reduceFn, final PType<V> valueType) {
  return new DetachingDoFn<K, V, T>(reduceFn, valueType);
}