本文整理汇总了Java中org.apache.crunch.impl.mem.MemPipeline.typedCollectionOf方法的典型用法代码示例。如果您正苦于以下问题：Java MemPipeline.typedCollectionOf方法的具体用法？Java MemPipeline.typedCollectionOf怎么用？Java MemPipeline.typedCollectionOf使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.crunch.impl.mem.MemPipeline的用法示例。
在下文中一共展示了MemPipeline.typedCollectionOf方法的10个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: testCategorical
import org.apache.crunch.impl.mem.MemPipeline; //导入方法依赖的package包/类
@Test
public void testCategorical() {
PCollection<String> input = MemPipeline.typedCollectionOf(
Avros.strings(),
"1.0,a,3.0,y",
"0.4,b,1.0,x",
"3.2,c,29.0,z");
PCollection<Record> elems = StringSplitFn.apply(input);
Summary s = new Summarizer()
.categoricalColumns(1, 3)
.build(elems).getValue();
PCollection<RealVector> vecs = elems.parallelDo(new StandardizeFn(s), MLAvros.vector());
assertEquals(ImmutableList.of(
Vectors.of(1.0, 1, 0, 0, 3.0, 0.0, 1.0, 0.0),
Vectors.of(0.4, 0, 1, 0, 1.0, 1.0, 0.0, 0.0),
Vectors.of(3.2, 0, 0, 1, 29.0, 0, 0, 1)),
vecs.materialize());
}
示例2: testKeyByAvroFieldSimple
import org.apache.crunch.impl.mem.MemPipeline; //导入方法依赖的package包/类
@Test
public void testKeyByAvroFieldSimple() throws PlanTimeException {
TestAvroRecord rec = TestAvroRecord.newBuilder().setFieldA(new Utf8("hello")).setFieldB("world").setFieldC(10L).build();
PCollection<TestAvroRecord> collection =
MemPipeline.typedCollectionOf(specifics(TestAvroRecord.class), rec);
PTable<String, TestAvroRecord> table = AvroCollections.keyByAvroField(collection, "fieldA", strings());
assertEquals(ImmutableMap.of("hello", rec), table.materializeToMap());
}
示例3: testKeyByAvroFieldNested
import org.apache.crunch.impl.mem.MemPipeline; //导入方法依赖的package包/类
@Test
public void testKeyByAvroFieldNested() throws PlanTimeException {
TestAvroRecord rec = TestAvroRecord.newBuilder().setFieldA(new Utf8("hello")).setFieldB("world").setFieldC(10L).build();
NestAvroRecord nest = NestAvroRecord.newBuilder().setFieldX("eggs").setFieldY(rec).build();
PCollection<NestAvroRecord> collection =
MemPipeline.typedCollectionOf(specifics(NestAvroRecord.class), nest);
PTable<String, NestAvroRecord> table = AvroCollections.keyByAvroField(collection, "fieldY.fieldA", strings());
assertEquals(ImmutableMap.of("hello", nest), table.materializeToMap());
}
示例4: ImmutableLists
import org.apache.crunch.impl.mem.MemPipeline; //导入方法依赖的package包/类
@Test
@Ignore("Guava ImmutableLists (which back MemCollections) cannot contain nulls")
public void testExtractNull() throws PlanTimeException {
TestAvroRecord rec = TestAvroRecord.newBuilder().setFieldA(new Utf8("hello")).setFieldB(null).setFieldC(10L).build();
PCollection<TestAvroRecord> collection =
MemPipeline.typedCollectionOf(specifics(TestAvroRecord.class), rec);
PCollection<String> result = AvroCollections.extract(collection, "fieldB", strings());
assertEquals(null, result.materialize().iterator().next());
}
示例5: testExtract2
import org.apache.crunch.impl.mem.MemPipeline; //导入方法依赖的package包/类
@Test
public void testExtract2() {
TestAvroRecord rec = TestAvroRecord.newBuilder().setFieldA(new Utf8("hello")).setFieldB("world").setFieldC(10L).build();
PCollection<TestAvroRecord> collection =
MemPipeline.typedCollectionOf(specifics(TestAvroRecord.class), rec);
PTable<String, String> table = AvroCollections.extract(collection, "fieldA", "fieldB", tableOf(strings(), strings()));
assertEquals(ImmutableMap.of("hello", "world"), table.materializeToMap());
}
示例6: testExtract3
import org.apache.crunch.impl.mem.MemPipeline; //导入方法依赖的package包/类
@Test
public void testExtract3() {
TestAvroRecord rec = TestAvroRecord.newBuilder().setFieldA(new Utf8("hello")).setFieldB("world").setFieldC(10L).build();
NestAvroRecord nest = NestAvroRecord.newBuilder().setFieldX("eggs").setFieldY(rec).build();
PCollection<NestAvroRecord> collection =
MemPipeline.typedCollectionOf(specifics(NestAvroRecord.class), nest);
PCollection<Tuple3<String, String, String>> coll =
AvroCollections.extract(collection, "fieldY.fieldA", "fieldY.fieldB", "fieldX", triples(strings(), strings(), strings()));
Tuple3<String, String, String> actual = coll.materialize().iterator().next();
assertEquals(Tuple3.of("hello", "world", "eggs"), actual);
}
示例7: testGlobalToplist
import org.apache.crunch.impl.mem.MemPipeline; //导入方法依赖的package包/类
@Test
public void testGlobalToplist() {
PCollection<String> data = MemPipeline.typedCollectionOf(strings(), "a", "a", "a", "b", "b", "c", "c", "c", "c");
Map<String, Long> actual = TopLists.globalToplist(data).materializeToMap();
Map<String, Long> expected = ImmutableMap.of("c", 4L, "a", 3L, "b", 2L);
assertEquals(expected, actual);
}
示例8: testSimple
import org.apache.crunch.impl.mem.MemPipeline; //导入方法依赖的package包/类
@Test
public void testSimple() {
PCollection<String> input = MemPipeline.typedCollectionOf(
Avros.strings(),
"1.0,2.0,3.0",
"0.4,2.0,1.0",
"3.2,17.0,29.0");
PCollection<Record> elems = StringSplitFn.apply(input);
PCollection<RealVector> vecs = elems.parallelDo(new StandardizeFn(), MLAvros.vector());
assertEquals(ImmutableList.of(Vectors.of(1, 2, 3), Vectors.of(0.4, 2, 1),
Vectors.of(3.2, 17, 29)), vecs.materialize());
}
示例9: testQuoted
import org.apache.crunch.impl.mem.MemPipeline; //导入方法依赖的package包/类
@Test
public void testQuoted() {
PCollection<String> input = MemPipeline.typedCollectionOf(
Avros.strings(),
"1.0,\"a,b,c\",3.0,y",
"0.4,\"b,q\",1.0,x",
"3.2,\"c\",29.0,z");
PCollection<Record> elems = StringSplitFn.apply(input);
for (Record r : elems.materialize()) {
assertEquals(4, r.size());
}
}
示例10: testNegateCounts
import org.apache.crunch.impl.mem.MemPipeline; //导入方法依赖的package包/类
@Test
public void testNegateCounts() {
PCollection<String> data = MemPipeline.typedCollectionOf(strings(), "a", "a", "a", "b", "b");
Map<String, Long> actual = SPTables.negateCounts(data.count()).materializeToMap();
Map<String, Long> expected = ImmutableMap.of("a", -3L, "b", -2L);
assertEquals(expected, actual);
}