Java MemPipeline类代码示例

import org.apache.crunch.impl.mem.MemPipeline; //导入依赖的package包/类
@Test
public void testPercentilesExact() {
  PTable<String, Integer> testTable = MemPipeline.typedTableOf(
          tableOf(strings(), ints()),
          "a", 5,
          "a", 2,
          "a", 3,
          "a", 4,
          "a", 1);
  Map<String, Result<Integer>> actualS = Percentiles.distributed(testTable, 0, 0.5, 1.0).materializeToMap();
  Map<String, Result<Integer>> actualM = Percentiles.inMemory(testTable, 0, 0.5, 1.0).materializeToMap();
  Map<String, Result<Integer>> expected = ImmutableMap.of(
          "a", result(5, Pair.of(0.0, 1), Pair.of(0.5, 3), Pair.of(1.0, 5))
  );

  assertEquals(expected, actualS);
  assertEquals(expected, actualM);
}
 

import org.apache.crunch.impl.mem.MemPipeline; //导入依赖的package包/类
@Test
public void testPercentilesBetween() {
  PTable<String, Integer> testTable = MemPipeline.typedTableOf(
          tableOf(strings(), ints()),
          "a", 5,
          "a", 2, // We expect the 0.5 to correspond to this element, according to the "nearest rank" %ile definition.
          "a", 4,
          "a", 1);
  Map<String, Result<Integer>> actualS = Percentiles.distributed(testTable, 0.5).materializeToMap();
  Map<String, Result<Integer>> actualM = Percentiles.inMemory(testTable, 0.5).materializeToMap();
  Map<String, Result<Integer>> expected = ImmutableMap.of(
          "a", result(4, Pair.of(0.5, 2))
  );

  assertEquals(expected, actualS);
  assertEquals(expected, actualM);
}
 

import org.apache.crunch.impl.mem.MemPipeline; //导入依赖的package包/类
@Test
public void testPercentilesNines() {
  PTable<String, Integer> testTable = MemPipeline.typedTableOf(
          tableOf(strings(), ints()),
          "a", 10,
          "a", 20,
          "a", 30,
          "a", 40,
          "a", 50,
          "a", 60,
          "a", 70,
          "a", 80,
          "a", 90,
          "a", 100);
  Map<String, Result<Integer>> actualS = Percentiles.distributed(testTable, 0.9, 0.99).materializeToMap();
  Map<String, Result<Integer>> actualM = Percentiles.inMemory(testTable, 0.9, 0.99).materializeToMap();
  Map<String, Result<Integer>> expected = ImmutableMap.of(
          "a", result(10, Pair.of(0.9, 90), Pair.of(0.99, 100))
  );

  assertEquals(expected, actualS);
  assertEquals(expected, actualM);
}
 

import org.apache.crunch.impl.mem.MemPipeline; //导入依赖的package包/类
@Test
public void testPercentilesLessThanOrEqual() {
  PTable<String, Integer> testTable = MemPipeline.typedTableOf(
          tableOf(strings(), ints()),
          "a", 10,
          "a", 20,
          "a", 30,
          "a", 40,
          "a", 50,
          "a", 60,
          "a", 70,
          "a", 80,
          "a", 90,
          "a", 100);
  Map<String, Result<Integer>> actualS = Percentiles.distributed(testTable, 0.5).materializeToMap();
  Map<String, Result<Integer>> actualM = Percentiles.inMemory(testTable, 0.5).materializeToMap();
  Map<String, Result<Integer>> expected = ImmutableMap.of(
          "a", result(10, Pair.of(0.5, 50))
  );

  assertEquals(expected, actualS);
  assertEquals(expected, actualM);
}
 

import org.apache.crunch.impl.mem.MemPipeline; //导入依赖的package包/类
@Test
public void testMeanValue() {
  PTable<String, Integer> testTable = MemPipeline.typedTableOf(
          tableOf(strings(), ints()),
          "a", 2,
          "a", 10,
          "b", 3,
          "c", 3,
          "c", 4,
          "c", 5);
  Map<String, Double> actual = Averages.meanValue(testTable).materializeToMap();
  Map<String, Double> expected = ImmutableMap.of(
          "a", 6.0,
          "b", 3.0,
          "c", 4.0
  );

  assertEquals(expected, actual);
}
 

import org.apache.crunch.impl.mem.MemPipeline; //导入依赖的package包/类
@Test
public void testTopNYbyX() {
  PTable<String, String> data = MemPipeline.typedTableOf(tableOf(strings(), strings()),
          "a","x",
          "a","x",
          "a","x",
          "a","y",
          "a","y",
          "a","z",
          "b","x",
          "b","x",
          "b","z");
  Map<String, Collection<Pair<Long, String>>> actual = TopLists.topNYbyX(data, 2).materializeToMap();
  Map<String, Collection<Pair<Long, String>>> expected = ImmutableMap.of(
          "a", collectionOf(Pair.of(3L, "x"), Pair.of(2L, "y")),
          "b", collectionOf(Pair.of(2L, "x"), Pair.of(1L, "z")));

  assertEquals(expected, actual);
}
 

import org.apache.crunch.impl.mem.MemPipeline; //导入依赖的package包/类
@Test
public void testCategorical() {
  PCollection<String> input = MemPipeline.typedCollectionOf(
      Avros.strings(),
      "1.0,a,3.0,y",
      "0.4,b,1.0,x",
      "3.2,c,29.0,z");
  PCollection<Record> elems = StringSplitFn.apply(input);
  Summary s = new Summarizer()
    .categoricalColumns(1, 3)
    .build(elems).getValue();
  PCollection<RealVector> vecs = elems.parallelDo(new StandardizeFn(s), MLAvros.vector());
  assertEquals(ImmutableList.of(
      Vectors.of(1.0, 1, 0, 0, 3.0, 0.0, 1.0, 0.0),
      Vectors.of(0.4, 0, 1, 0, 1.0, 1.0, 0.0, 0.0),
      Vectors.of(3.2, 0, 0, 1, 29.0, 0, 0, 1)),
      vecs.materialize());
}
 

import org.apache.crunch.impl.mem.MemPipeline; //导入依赖的package包/类
@Test
public void testSwapKeyValue() {
  PTable<String, Long> table = MemPipeline.typedTableOf(tableOf(strings(), longs()), "hello", 14L, "goodbye", 21L);
  PTable<Long, String> actual = SPTables.swapKeyValue(table);
  Map<Long, String> expected = ImmutableMap.of(14L, "hello", 21L, "goodbye");
  assertEquals(expected, actual.materializeToMap());
}
 

import org.apache.crunch.impl.mem.MemPipeline; //导入依赖的package包/类
@Test
public void testKeyByAvroFieldSimple() throws PlanTimeException {
  TestAvroRecord rec = TestAvroRecord.newBuilder().setFieldA(new Utf8("hello")).setFieldB("world").setFieldC(10L).build();
  PCollection<TestAvroRecord> collection =
          MemPipeline.typedCollectionOf(specifics(TestAvroRecord.class), rec);

  PTable<String, TestAvroRecord> table = AvroCollections.keyByAvroField(collection, "fieldA", strings());
  assertEquals(ImmutableMap.of("hello", rec), table.materializeToMap());
}
 

import org.apache.crunch.impl.mem.MemPipeline; //导入依赖的package包/类
@Test
public void testKeyByAvroFieldNested() throws PlanTimeException {
  TestAvroRecord rec = TestAvroRecord.newBuilder().setFieldA(new Utf8("hello")).setFieldB("world").setFieldC(10L).build();
  NestAvroRecord nest = NestAvroRecord.newBuilder().setFieldX("eggs").setFieldY(rec).build();
  PCollection<NestAvroRecord> collection =
          MemPipeline.typedCollectionOf(specifics(NestAvroRecord.class), nest);

  PTable<String, NestAvroRecord> table = AvroCollections.keyByAvroField(collection, "fieldY.fieldA", strings());
  assertEquals(ImmutableMap.of("hello", nest), table.materializeToMap());
}
 

import org.apache.crunch.impl.mem.MemPipeline; //导入依赖的package包/类
@Test
@Ignore("Guava ImmutableLists (which back MemCollections) cannot contain nulls")
public void testExtractNull() throws PlanTimeException {
  TestAvroRecord rec = TestAvroRecord.newBuilder().setFieldA(new Utf8("hello")).setFieldB(null).setFieldC(10L).build();
  PCollection<TestAvroRecord> collection =
          MemPipeline.typedCollectionOf(specifics(TestAvroRecord.class), rec);

  PCollection<String> result = AvroCollections.extract(collection, "fieldB", strings());
  assertEquals(null, result.materialize().iterator().next());
}
 

import org.apache.crunch.impl.mem.MemPipeline; //导入依赖的package包/类
@Test
public void testExtract2() {
  TestAvroRecord rec = TestAvroRecord.newBuilder().setFieldA(new Utf8("hello")).setFieldB("world").setFieldC(10L).build();
  PCollection<TestAvroRecord> collection =
          MemPipeline.typedCollectionOf(specifics(TestAvroRecord.class), rec);

  PTable<String, String> table = AvroCollections.extract(collection, "fieldA", "fieldB", tableOf(strings(), strings()));
  assertEquals(ImmutableMap.of("hello", "world"), table.materializeToMap());
}
 

import org.apache.crunch.impl.mem.MemPipeline; //导入依赖的package包/类
@Test
public void testExtract3() {
  TestAvroRecord rec = TestAvroRecord.newBuilder().setFieldA(new Utf8("hello")).setFieldB("world").setFieldC(10L).build();
  NestAvroRecord nest = NestAvroRecord.newBuilder().setFieldX("eggs").setFieldY(rec).build();
  PCollection<NestAvroRecord> collection =
          MemPipeline.typedCollectionOf(specifics(NestAvroRecord.class), nest);
  PCollection<Tuple3<String, String, String>> coll =
          AvroCollections.extract(collection, "fieldY.fieldA", "fieldY.fieldB", "fieldX", triples(strings(), strings(), strings()));
  Tuple3<String, String, String> actual = coll.materialize().iterator().next();
  assertEquals(Tuple3.of("hello", "world", "eggs"), actual);
}
 

import org.apache.crunch.impl.mem.MemPipeline; //导入依赖的package包/类
@Test
public void testGlobalToplist() {
  PCollection<String> data = MemPipeline.typedCollectionOf(strings(), "a", "a", "a", "b", "b", "c", "c", "c", "c");
  Map<String, Long> actual = TopLists.globalToplist(data).materializeToMap();
  Map<String, Long> expected = ImmutableMap.of("c", 4L, "a", 3L, "b", 2L);
  assertEquals(expected, actual);
}
 

import org.apache.crunch.impl.mem.MemPipeline; //导入依赖的package包/类
@Test
public void testSimple() {
  PCollection<String> input = MemPipeline.typedCollectionOf(
      Avros.strings(),
      "1.0,2.0,3.0",
      "0.4,2.0,1.0",
      "3.2,17.0,29.0");
  PCollection<Record> elems = StringSplitFn.apply(input);
  PCollection<RealVector> vecs = elems.parallelDo(new StandardizeFn(), MLAvros.vector());
  assertEquals(ImmutableList.of(Vectors.of(1, 2, 3), Vectors.of(0.4, 2, 1),
      Vectors.of(3.2, 17, 29)), vecs.materialize());
}