本文整理汇总了Java中org.apache.hadoop.mapreduce.lib.fieldsel.TestMRFieldSelection.constructInputOutputData方法的典型用法代码示例。如果您正苦于以下问题：Java TestMRFieldSelection.constructInputOutputData方法的具体用法？Java TestMRFieldSelection.constructInputOutputData怎么用？Java TestMRFieldSelection.constructInputOutputData使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.hadoop.mapreduce.lib.fieldsel.TestMRFieldSelection的用法示例。
在下文中一共展示了TestMRFieldSelection.constructInputOutputData方法的2个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: launch
import org.apache.hadoop.mapreduce.lib.fieldsel.TestMRFieldSelection; //导入方法依赖的package包/类
public static void launch() throws Exception {
JobConf conf = new JobConf(TestFieldSelection.class);
FileSystem fs = FileSystem.get(conf);
int numOfInputLines = 10;
Path OUTPUT_DIR = new Path("build/test/output_for_field_selection_test");
Path INPUT_DIR = new Path("build/test/input_for_field_selection_test");
String inputFile = "input.txt";
fs.delete(INPUT_DIR, true);
fs.mkdirs(INPUT_DIR);
fs.delete(OUTPUT_DIR, true);
StringBuffer inputData = new StringBuffer();
StringBuffer expectedOutput = new StringBuffer();
TestMRFieldSelection.constructInputOutputData(inputData,
expectedOutput, numOfInputLines);
FSDataOutputStream fileOut = fs.create(new Path(INPUT_DIR, inputFile));
fileOut.write(inputData.toString().getBytes("utf-8"));
fileOut.close();
System.out.println("inputData:");
System.out.println(inputData.toString());
JobConf job = new JobConf(conf, TestFieldSelection.class);
FileInputFormat.setInputPaths(job, INPUT_DIR);
job.setInputFormat(TextInputFormat.class);
job.setMapperClass(FieldSelectionMapReduce.class);
job.setReducerClass(FieldSelectionMapReduce.class);
FileOutputFormat.setOutputPath(job, OUTPUT_DIR);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setOutputFormat(TextOutputFormat.class);
job.setNumReduceTasks(1);
job.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-");
job.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-");
job.set(FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-");
JobClient.runJob(job);
//
// Finally, we compare the reconstructed answer key with the
// original one. Remember, we need to ignore zero-count items
// in the original key.
//
boolean success = true;
Path outPath = new Path(OUTPUT_DIR, "part-00000");
String outdata = MapReduceTestUtil.readOutput(outPath,job);
assertEquals(expectedOutput.toString(),outdata);
fs.delete(OUTPUT_DIR, true);
fs.delete(INPUT_DIR, true);
}
示例2: launch
import org.apache.hadoop.mapreduce.lib.fieldsel.TestMRFieldSelection; //导入方法依赖的package包/类
public static void launch() throws Exception {
JobConf conf = new JobConf(TestFieldSelection.class);
FileSystem fs = FileSystem.get(conf);
int numOfInputLines = 10;
String baseDir = System.getProperty("test.build.data", "build/test/data");
Path OUTPUT_DIR = new Path(baseDir + "/output_for_field_selection_test");
Path INPUT_DIR = new Path(baseDir + "/input_for_field_selection_test");
String inputFile = "input.txt";
fs.delete(INPUT_DIR, true);
fs.mkdirs(INPUT_DIR);
fs.delete(OUTPUT_DIR, true);
StringBuffer inputData = new StringBuffer();
StringBuffer expectedOutput = new StringBuffer();
TestMRFieldSelection.constructInputOutputData(inputData,
expectedOutput, numOfInputLines);
FSDataOutputStream fileOut = fs.create(new Path(INPUT_DIR, inputFile));
fileOut.write(inputData.toString().getBytes("utf-8"));
fileOut.close();
System.out.println("inputData:");
System.out.println(inputData.toString());
JobConf job = new JobConf(conf, TestFieldSelection.class);
FileInputFormat.setInputPaths(job, INPUT_DIR);
job.setInputFormat(TextInputFormat.class);
job.setMapperClass(FieldSelectionMapReduce.class);
job.setReducerClass(FieldSelectionMapReduce.class);
FileOutputFormat.setOutputPath(job, OUTPUT_DIR);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(Text.class);
job.setOutputFormat(TextOutputFormat.class);
job.setNumReduceTasks(1);
job.set(FieldSelectionHelper.DATA_FIELD_SEPERATOR, "-");
job.set(FieldSelectionHelper.MAP_OUTPUT_KEY_VALUE_SPEC, "6,5,1-3:0-");
job.set(FieldSelectionHelper.REDUCE_OUTPUT_KEY_VALUE_SPEC, ":4,3,2,1,0,0-");
JobClient.runJob(job);
//
// Finally, we compare the reconstructed answer key with the
// original one. Remember, we need to ignore zero-count items
// in the original key.
//
boolean success = true;
Path outPath = new Path(OUTPUT_DIR, "part-00000");
String outdata = MapReduceTestUtil.readOutput(outPath,job);
assertEquals(expectedOutput.toString(),outdata);
fs.delete(OUTPUT_DIR, true);
fs.delete(INPUT_DIR, true);
}