当前位置: 首页>>代码示例>>Java>>正文

Java KeyValueTextInputFormat类代码示例

本文整理汇总了Java中org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat类的典型用法代码示例。如果您正苦于以下问题：Java KeyValueTextInputFormat类的具体用法？Java KeyValueTextInputFormat怎么用？Java KeyValueTextInputFormat使用的例子？那么, 这里精选的类代码示例或许可以为您提供帮助。


KeyValueTextInputFormat类属于org.apache.hadoop.mapreduce.lib.input包，在下文中一共展示了KeyValueTextInputFormat类的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: testAddInputPathWithMapper

import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; //导入依赖的package包/类
@SuppressWarnings("unchecked")
public void testAddInputPathWithMapper() throws IOException {
  final Job conf = Job.getInstance();
  MultipleInputs.addInputPath(conf, new Path("/foo"), TextInputFormat.class,
     MapClass.class);
  MultipleInputs.addInputPath(conf, new Path("/bar"),
      KeyValueTextInputFormat.class, KeyValueMapClass.class);
  final Map<Path, InputFormat> inputs = MultipleInputs
     .getInputFormatMap(conf);
  final Map<Path, Class<? extends Mapper>> maps = MultipleInputs
     .getMapperTypeMap(conf);

  assertEquals(TextInputFormat.class, inputs.get(new Path("/foo")).getClass());
  assertEquals(KeyValueTextInputFormat.class, inputs.get(new Path("/bar"))
     .getClass());
  assertEquals(MapClass.class, maps.get(new Path("/foo")));
  assertEquals(KeyValueMapClass.class, maps.get(new Path("/bar")));
}
开发者ID:naver，项目名称:hadoop，代码行数:19，代码来源:TestMultipleInputs.java

示例2: jobListFriends

import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; //导入依赖的package包/类
private Job jobListFriends(String inputPath, String outputPath) throws IOException, InterruptedException, ClassNotFoundException{      
    Job job = new Job();
    job.setJarByClass(WordCount.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setMapperClass(Map.class);
    job.setReducerClass(Reduce.class);

    job.setInputFormatClass(KeyValueTextInputFormat.class);   // Need to change the import
    job.setOutputFormatClass(TextOutputFormat.class);

    FileInputFormat.addInputPath(job, new Path(inputPath));
    FileOutputFormat.setOutputPath(job, new Path(outputPath));

    job.waitForCompletion(true);

    return job;
}
开发者ID:dhruvmalik007，项目名称:Deep_learning_using_Java，代码行数:20，代码来源:Recommendation_program.java

示例3: jobRecommendFriends

import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; //导入依赖的package包/类
private Job jobRecommendFriends(String inputPath, String outputPath) throws IOException, InterruptedException, ClassNotFoundException{     
    Job job1 = new Job();
    job1.setJarByClass(WordCount.class);
    job1.setOutputKeyClass(Text.class);
    job1.setOutputValueClass(Text.class);
   
    job1.setMapperClass(MapRecommendation.class);
    job1.setReducerClass(ReduceRecommendation.class);
   
    job1.setOutputFormatClass(TextOutputFormat.class);
    job1.setInputFormatClass(KeyValueTextInputFormat.class);

    FileInputFormat.addInputPath(job1, new Path(inputPath));
    FileOutputFormat.setOutputPath(job1, new Path(outputPath));

    job1.waitForCompletion(true);

    return job1;
   
}
开发者ID:dhruvmalik007，项目名称:Deep_learning_using_Java，代码行数:21，代码来源:Recommendation_program.java

示例4: testAddInputPathWithMapper

import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; //导入依赖的package包/类
@SuppressWarnings("unchecked")
public void testAddInputPathWithMapper() throws IOException {
  final Job conf = new Job();
  MultipleInputs.addInputPath(conf, new Path("/foo"), TextInputFormat.class,
     MapClass.class);
  MultipleInputs.addInputPath(conf, new Path("/bar"),
      KeyValueTextInputFormat.class, KeyValueMapClass.class);
  final Map<Path, InputFormat> inputs = MultipleInputs
     .getInputFormatMap(conf);
  final Map<Path, Class<? extends Mapper>> maps = MultipleInputs
     .getMapperTypeMap(conf);

  assertEquals(TextInputFormat.class, inputs.get(new Path("/foo")).getClass());
  assertEquals(KeyValueTextInputFormat.class, inputs.get(new Path("/bar"))
     .getClass());
  assertEquals(MapClass.class, maps.get(new Path("/foo")));
  assertEquals(KeyValueMapClass.class, maps.get(new Path("/bar")));
}
开发者ID:Nextzero，项目名称:hadoop-2.6.0-cdh5.4.3，代码行数:19，代码来源:TestMultipleInputs.java

示例5: runCalcJob

import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; //导入依赖的package包/类
public static boolean runCalcJob(Configuration conf, Path input, Path outputPath)
    throws Exception {

  Job job = new Job(conf);
  job.setJarByClass(Main.class);
  job.setMapperClass(CalcMapReduce.Map.class);
  job.setReducerClass(CalcMapReduce.Reduce.class);

  job.setInputFormatClass(KeyValueTextInputFormat.class);

  job.setMapOutputKeyClass(CalcMapReduce.TextPair.class);
  job.setMapOutputValueClass(IntWritable.class);

  FileInputFormat.setInputPaths(job, input);
  FileOutputFormat.setOutputPath(job, outputPath);

  return job.waitForCompletion(true);
}
开发者ID:Hanmourang，项目名称:hiped2，代码行数:19，代码来源:Main.java

示例6: runSortJob

import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; //导入依赖的package包/类
public static void runSortJob(Configuration conf, Path input, Path outputPath)
    throws Exception {

  Job job = new Job(conf);
  job.setJarByClass(Main.class);
  job.setMapperClass(SortMapReduce.Map.class);
  job.setReducerClass(SortMapReduce.Reduce.class);

  job.setInputFormatClass(KeyValueTextInputFormat.class);

  job.setMapOutputKeyClass(Person.class);
  job.setMapOutputValueClass(Person.class);

  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(Text.class);

  job.setPartitionerClass(PersonNamePartitioner.class);
  job.setSortComparatorClass(PersonComparator.class);
  job.setGroupingComparatorClass(PersonNameComparator.class);

  FileInputFormat.setInputPaths(job, input);
  FileOutputFormat.setOutputPath(job, outputPath);

  job.waitForCompletion(true);
}
开发者ID:Hanmourang，项目名称:hiped2，代码行数:26，代码来源:Main.java

示例7: runJob

import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; //导入依赖的package包/类
public static void runJob(Configuration conf,
                          Path inputPath,
                          Path outputPath)
    throws Exception {

  Job job = new Job(conf);

  job.setJarByClass(UniqueHashedKeyJob.class);
  job.setMapperClass(Map.class);
  job.setReducerClass(Reduce.class);

  job.setInputFormatClass(KeyValueTextInputFormat.class);
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(NullWritable.class);
  job.setOutputKeyClass(Text.class);
  job.setOutputValueClass(NullWritable.class);

  outputPath.getFileSystem(conf).delete(outputPath, true);

  FileInputFormat.setInputPaths(job, inputPath);
  FileOutputFormat.setOutputPath(job, outputPath);

  if (!job.waitForCompletion(true)) {
    throw new Exception("Job failed");
  }
}
开发者ID:Hanmourang，项目名称:hiped2，代码行数:27，代码来源:UniqueHashedKeyJob.java

示例8: startSSSPJob

import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; //导入依赖的package包/类
private static void startSSSPJob(String inputPath, String outputPath) throws IOException, InterruptedException, ClassNotFoundException {
	Configuration conf = new Configuration();
	Job job = new Job(conf, "SSSP Job");

	job.setJarByClass(Driver.class);
	job.setMapperClass(SSSPMapper.class);
	job.setReducerClass(SSSPReducer.class);

	job.setNumReduceTasks(1);

	job.setOutputKeyClass(LongWritable.class);
	job.setOutputValueClass(Text.class);

	job.setInputFormatClass(KeyValueTextInputFormat.class);
	job.setOutputFormatClass(TextOutputFormat.class);

	TextInputFormat.addInputPath(job, new Path(inputPath));
	TextOutputFormat.setOutputPath(job, new Path(outputPath));

	job.waitForCompletion(true);
}
开发者ID:tayllan，项目名称:comparative-study-of-frameworks-for-parallel-processing-of-graphs，代码行数:22，代码来源:Driver.java

示例9: testAddInputPathWithMapper

import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; //导入依赖的package包/类
@Test
public void testAddInputPathWithMapper() throws IOException {
  final Job conf = Job.getInstance();
  MultipleInputs.addInputPath(conf, new Path("/foo"), TextInputFormat.class,
     MapClass.class);
  MultipleInputs.addInputPath(conf, new Path("/bar"),
      KeyValueTextInputFormat.class, KeyValueMapClass.class);
  final Map<Path, InputFormat> inputs = MultipleInputs
     .getInputFormatMap(conf);
  final Map<Path, Class<? extends Mapper>> maps = MultipleInputs
     .getMapperTypeMap(conf);

  assertEquals(TextInputFormat.class, inputs.get(new Path("/foo")).getClass());
  assertEquals(KeyValueTextInputFormat.class, inputs.get(new Path("/bar"))
     .getClass());
  assertEquals(MapClass.class, maps.get(new Path("/foo")));
  assertEquals(KeyValueMapClass.class, maps.get(new Path("/bar")));
}
开发者ID:hopshadoop，项目名称:hops，代码行数:19，代码来源:TestMultipleInputs.java

示例10: run

import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; //导入依赖的package包/类
public int run(String[] args) throws Exception {

    if (args.length != 2) {
      System.out.printf("Usage: InvertedIndex <input dir> <output dir>\n");
      return -1;
    }

    Job job = new Job(getConf());
    job.setJarByClass(InvertedIndex.class);
    job.setJobName("Inverted Index");

    FileInputFormat.setInputPaths(job, new Path(args[0]));
    FileOutputFormat.setOutputPath(job, new Path(args[1]));
    job.setInputFormatClass(KeyValueTextInputFormat.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    
    job.setMapperClass(IndexMapper.class);
    job.setReducerClass(IndexReducer.class);

    boolean success = job.waitForCompletion(true);
    return success ? 0 : 1;
  }
开发者ID:mellowonpsx，项目名称:cloudera-homework，代码行数:24，代码来源:InvertedIndex.java

示例11: main

import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; //导入依赖的package包/类
public static void main(String[] args) throws Exception {

        Job job = Job.getInstance(new Configuration());
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);

        job.setMapOutputKeyClass(NullWritable.class);
        job.setMapOutputValueClass(TextArrayWritable.class);

        job.setMapperClass(TopWordsMap.class);
        job.setReducerClass(TopWordsReduce.class);
        job.setNumReduceTasks(1);

        FileInputFormat.setInputPaths(job, new Path(args[0]));
        FileOutputFormat.setOutputPath(job, new Path(args[1]));

        job.setInputFormatClass(KeyValueTextInputFormat.class);
        job.setOutputFormatClass(TextOutputFormat.class);

        job.setJarByClass(TopWords2.class);
        System.exit(job.waitForCompletion(true) ? 0 : 1);

    }
开发者ID:xldrx，项目名称:mapreduce_examples，代码行数:24，代码来源:TopWords2.java

示例12: main

import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
	Configuration conf = new Configuration();
	Job job = new Job(conf, "tweetLanguage");
	job.setJarByClass(TweetLanguage.class);
	job.setMapperClass(TweetLanguageMapper.class);
	job.setReducerClass(MapReducer.class);
	job.setOutputKeyClass(Text.class);
	job.setOutputValueClass(Text.class);
	job.setInputFormatClass(KeyValueTextInputFormat.class);
	FileInputFormat
			.addInputPath(
					job,
					new Path("/path/to/twitter_sample.*.queue"));
	FileOutputFormat.setOutputPath(job, new Path("output_tweetLang_clean"));
	System.exit(job.waitForCompletion(true) ? 0 : 1);
}
开发者ID:computermacgyver，项目名称:twitter-mapred，代码行数:17，代码来源:TweetLanguage.java

示例13: main

import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
	Configuration conf = new Configuration();
	Job job = new Job(conf, "userDescriptionLanguage");
	job.setJarByClass(UserDescriptionLanguage.class);
	job.setMapperClass(UserDescriptionLanguageMapper.class);
	job.setReducerClass(MapReducer.class);
	job.setOutputKeyClass(Text.class);
	job.setOutputValueClass(Text.class);
	job.setInputFormatClass(KeyValueTextInputFormat.class);
	FileInputFormat
			.addInputPath(
					job,
					new Path("/path/to/twitter_sample.*.queue"));
	FileOutputFormat.setOutputPath(job, new Path("output_userLang"));
	System.exit(job.waitForCompletion(true) ? 0 : 1);
}
开发者ID:computermacgyver，项目名称:twitter-mapred，代码行数:17，代码来源:UserDescriptionLanguage.java

示例14: main

import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
	Configuration conf = new Configuration();
	Job job = new Job(conf, "userMentions");
	job.setJarByClass(UserMentions.class);
	job.setMapperClass(UserMentionMapper.class);
	job.setReducerClass(SumReducer.class);
	job.setOutputKeyClass(Text.class);
	job.setOutputValueClass(IntWritable.class);
	job.setInputFormatClass(KeyValueTextInputFormat.class);
	FileInputFormat
			.addInputPath(
					job,
					new Path("/path/to/twitter_sample.*.queue"));
	FileOutputFormat.setOutputPath(job, new Path("output_userMentions"));
	System.exit(job.waitForCompletion(true) ? 0 : 1);
}
开发者ID:computermacgyver，项目名称:twitter-mapred，代码行数:17，代码来源:UserMentions.java

示例15: main

import org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
	Configuration conf = new Configuration();
	Job job = new Job(conf, "userMentions");
	job.setJarByClass(GeotagExtract.class);
	job.setMapperClass(GeotagMapper.class);
	//job.setReducerClass(...);//Identity reducer
	job.setOutputKeyClass(Text.class);
	job.setOutputValueClass(Text.class);
	job.setInputFormatClass(KeyValueTextInputFormat.class);
	FileInputFormat
			.addInputPath(
					job,
					new Path("/path/to/twitter_sample.*.queue"));
	FileOutputFormat.setOutputPath(job, new Path("output_geotag"));
	System.exit(job.waitForCompletion(true) ? 0 : 1);
}
开发者ID:computermacgyver，项目名称:twitter-mapred，代码行数:17，代码来源:GeotagExtract.java


注：本文中的org.apache.hadoop.mapreduce.lib.input.KeyValueTextInputFormat类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。