本文整理汇总了Java中org.htuple.ShuffleUtils类的典型用法代码示例。如果您正苦于以下问题：Java ShuffleUtils类的具体用法？Java ShuffleUtils怎么用？Java ShuffleUtils使用的例子？那么, 这里精选的类代码示例或许可以为您提供帮助。
ShuffleUtils类属于org.htuple包，在下文中一共展示了ShuffleUtils类的9个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: run
import org.htuple.ShuffleUtils; //导入依赖的package包/类
/**
* The MapReduce driver - setup and launch the job.
*
* @param args the command-line arguments
* @return the process exit code
* @throws Exception if something goes wrong
*/
public int run(final String[] args) throws Exception {
Cli cli = Cli.builder().setArgs(args).addOptions(Options.values()).build();
int result = cli.runCmd();
if (result != 0) {
return result;
}
Path usersPath = new Path(cli.getArgValueAsString(Options.USERS));
Path userLogsPath = new Path(cli.getArgValueAsString(Options.USER_LOGS));
Path outputPath = new Path(cli.getArgValueAsString(Options.OUTPUT));
Configuration conf = super.getConf();
Job job = new Job(conf);
job.setJarByClass(StreamingRepartitionJoin.class);
MultipleInputs.addInputPath(job, usersPath, TextInputFormat.class, UserMap.class);
MultipleInputs.addInputPath(job, userLogsPath, TextInputFormat.class, UserLogMap.class);
ShuffleUtils.configBuilder()
.useNewApi()
.setSortIndices(KeyFields.USER, KeyFields.DATASET)
.setPartitionerIndices(KeyFields.USER)
.setGroupIndices(KeyFields.USER)
.configure(job.getConfiguration());
job.setReducerClass(Reduce.class);
job.setMapOutputKeyClass(Tuple.class);
job.setMapOutputValueClass(Tuple.class);
FileOutputFormat.setOutputPath(job, outputPath);
return job.waitForCompletion(true) ? 0 : 1;
}
示例2: setupSecondarySort
import org.htuple.ShuffleUtils; //导入依赖的package包/类
/**
* Partition and group on just the last name; sort on both last and first name.
*
* @param conf the Hadoop config
*/
public static void setupSecondarySort(Configuration conf) {
ShuffleUtils.configBuilder()
.useNewApi()
.setPartitionerIndices(TupleFields.LAST_NAME)
.setSortIndices(TupleFields.values())
.setGroupIndices(TupleFields.LAST_NAME)
.configure(conf);
}
示例3: testDifferentPartitions
import org.htuple.ShuffleUtils; //导入依赖的package包/类
@Test
public void testDifferentPartitions() {
Tuple t1 = new Tuple();
Tuple t2 = new Tuple();
t1.add("alex").add(1).add(3);
t2.add("alex").add(2).add(2);
ShuffleUtils.configBuilder().setPartitionerIndices(0).configure(conf);
TuplePartitioner partitioner = new TuplePartitioner(conf);
assertEquals(partitioner.getPartition(t1, 10), partitioner.getPartition(t2, 10));
ShuffleUtils.configBuilder().setPartitionerIndices(1).configure(conf);
partitioner = new TuplePartitioner(conf);
assertNotSame(partitioner.getPartition(t1, 10), partitioner.getPartition(t2, 10));
ShuffleUtils.configBuilder().setPartitionerIndices(2).configure(conf);
partitioner = new TuplePartitioner(conf);
assertNotSame(partitioner.getPartition(t1, 10), partitioner.getPartition(t2, 10));
ShuffleUtils.configBuilder().setPartitionerIndices(0, 1).configure(conf);
partitioner = new TuplePartitioner(conf);
assertNotSame(partitioner.getPartition(t1, 10), partitioner.getPartition(t2, 10));
ShuffleUtils.configBuilder().setPartitionerIndices(0, 2).configure(conf);
partitioner = new TuplePartitioner(conf);
assertNotSame(partitioner.getPartition(t1, 10), partitioner.getPartition(t2, 10));
}
示例4: testIndexOutOfBounds
import org.htuple.ShuffleUtils; //导入依赖的package包/类
@Test
public void testIndexOutOfBounds() {
Tuple t1 = new Tuple();
Tuple t2 = new Tuple();
t1.add("alex").add(1).add(3);
t2.add("alex").add(2).add(2);
ShuffleUtils.configBuilder().setPartitionerIndices(0, 5).configure(conf);
TuplePartitioner partitioner = new TuplePartitioner(conf);
assertEquals(partitioner.getPartition(t1, 10), partitioner.getPartition(t2, 10));
}
示例5: getConfigName
import org.htuple.ShuffleUtils; //导入依赖的package包/类
@Override
public String getConfigName() {
return ShuffleUtils.SORTING_INDEXES_CONFIG_NAME;
}
示例6: setComparatorIndices
import org.htuple.ShuffleUtils; //导入依赖的package包/类
@Override
public void setComparatorIndices(Configuration conf, int... indices) {
ShuffleUtils.configBuilder().setSortIndices(indices).configure(conf);
}
示例7: getConfigName
import org.htuple.ShuffleUtils; //导入依赖的package包/类
@Override
public String getConfigName() {
return ShuffleUtils.GROUPING_INDEXES_CONFIG_NAME;
}
示例8: setComparatorIndices
import org.htuple.ShuffleUtils; //导入依赖的package包/类
@Override
public void setComparatorIndices(Configuration conf, int... indices) {
ShuffleUtils.configBuilder().setGroupIndices(indices).configure(conf);
}
示例9: testEmptyConfig
import org.htuple.ShuffleUtils; //导入依赖的package包/类
@Test(expected = IllegalStateException.class)
public void testEmptyConfig() {
conf.setStrings(ShuffleUtils.PARTITIONER_INDEXES_CONFIG_NAME, "");
new TuplePartitioner(conf);
}