当前位置: 首页>>代码示例>>Java>>正文

Java MultipleInputs.addInputPath方法代码示例

本文整理汇总了Java中org.apache.hadoop.mapreduce.lib.input.MultipleInputs.addInputPath方法的典型用法代码示例。如果您正苦于以下问题：Java MultipleInputs.addInputPath方法的具体用法？Java MultipleInputs.addInputPath怎么用？Java MultipleInputs.addInputPath使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.hadoop.mapreduce.lib.input.MultipleInputs的用法示例。


在下文中一共展示了MultipleInputs.addInputPath方法的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: main

import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {

        Configuration conf = new Configuration();
        conf.setFloat("beta", Float.parseFloat(args[3]));
        Job job = Job.getInstance(conf);
        job.setJarByClass(UnitSum.class);

        ChainMapper.addMapper(job, PassMapper.class, Object.class, Text.class, Text.class, DoubleWritable.class, conf);
        ChainMapper.addMapper(job, BetaMapper.class, Text.class, DoubleWritable.class, Text.class, DoubleWritable.class, conf);

        job.setReducerClass(SumReducer.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(DoubleWritable.class);

        MultipleInputs.addInputPath(job, new Path(args[0]), TextInputFormat.class, PassMapper.class);
        MultipleInputs.addInputPath(job, new Path(args[1]), TextInputFormat.class, BetaMapper.class);

        FileOutputFormat.setOutputPath(job, new Path(args[2]));
        job.waitForCompletion(true);
    }
开发者ID:yogykwan，项目名称:mapreduce-samples，代码行数:21，代码来源:UnitSum.java

示例2: main

import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
	Configuration conf = new Configuration();

	Job job = Job.getInstance(conf);
	job.setJarByClass(Multiplication.class);

	ChainMapper.addMapper(job, CooccurrenceMapper.class, LongWritable.class, Text.class, Text.class, Text.class, conf);
	ChainMapper.addMapper(job, RatingMapper.class, Text.class, Text.class, Text.class, Text.class, conf);

	job.setMapperClass(CooccurrenceMapper.class);
	job.setMapperClass(RatingMapper.class);

	job.setReducerClass(MultiplicationReducer.class);

	job.setMapOutputKeyClass(Text.class);
	job.setMapOutputValueClass(Text.class);
	job.setOutputKeyClass(Text.class);
	job.setOutputValueClass(DoubleWritable.class);

	MultipleInputs.addInputPath(job, new Path(args[0]), TextInputFormat.class, CooccurrenceMapper.class);
	MultipleInputs.addInputPath(job, new Path(args[1]), TextInputFormat.class, RatingMapper.class);

	TextOutputFormat.setOutputPath(job, new Path(args[2]));
	
	job.waitForCompletion(true);
}
开发者ID:yogykwan，项目名称:mapreduce-samples，代码行数:27，代码来源:Multiplication.java

示例3: main

import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {

        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);
        job.setJarByClass(UnitMultiplication.class);

        ChainMapper.addMapper(job, TransitionMapper.class, Object.class, Text.class, Text.class, Text.class, conf);
        ChainMapper.addMapper(job, PRMapper.class, Object.class, Text.class, Text.class, Text.class, conf);

        job.setReducerClass(MultiplicationReducer.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        MultipleInputs.addInputPath(job, new Path(args[0]), TextInputFormat.class, TransitionMapper.class);
        MultipleInputs.addInputPath(job, new Path(args[1]), TextInputFormat.class, PRMapper.class);

        FileOutputFormat.setOutputPath(job, new Path(args[2]));
        job.waitForCompletion(true);
    }
开发者ID:yogykwan，项目名称:mapreduce-samples，代码行数:21，代码来源:UnitMultiplication.java

示例4: main

import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {

        Configuration conf = new Configuration();
        conf.setFloat("beta", Float.parseFloat(args[3]));
        Job job = Job.getInstance(conf);
        job.setJarByClass(UnitMultiplication.class);

        ChainMapper.addMapper(job, TransitionMapper.class, Object.class, Text.class, Text.class, Text.class, conf);
        ChainMapper.addMapper(job, PRMapper.class, Object.class, Text.class, Text.class, Text.class, conf);

        job.setReducerClass(MultiplicationReducer.class);

        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        MultipleInputs.addInputPath(job, new Path(args[0]), TextInputFormat.class, TransitionMapper.class);
        MultipleInputs.addInputPath(job, new Path(args[1]), TextInputFormat.class, PRMapper.class);

        FileOutputFormat.setOutputPath(job, new Path(args[2]));
        job.waitForCompletion(true);
    }
开发者ID:yogykwan，项目名称:mapreduce-samples，代码行数:22，代码来源:UnitMultiplication.java

示例5: main

import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    Job job = new Job(conf, "Reduce-side join");
    job.setJarByClass(ReduceJoin.class);
    job.setReducerClass(ReduceJoinReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    MultipleInputs.addInputPath(job, new Path(args[0]), TextInputFormat.class, SalesRecordMapper.class) ;
    MultipleInputs.addInputPath(job, new Path(args[1]), TextInputFormat.class, AccountRecordMapper.class) ;
    //        FileOutputFormat.setOutputPath(job, new Path(args[2]));
    Path outputPath = new Path(args[2]);
    FileOutputFormat.setOutputPath(job, outputPath);
    outputPath.getFileSystem(conf).delete(outputPath);
    
    System.exit(job.waitForCompletion(true) ? 0 : 1);
}
开发者ID:PacktPublishing，项目名称:Data-Science-with-Hadoop，代码行数:17，代码来源:ReduceJoin.java

示例6: main

import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
    Configuration conf = new Configuration();
    Job job = Job.getInstance(conf, "book merger");

    job.setJarByClass(BookMerger.class);
    job.setCombinerClass(BookDataReducer.class);
    job.setReducerClass(BookDataReducer.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(BookMapWritable.class);

    FileOutputFormat.setOutputPath(job, new Path(args[0]));
    MultipleInputs.addInputPath(job, new Path(args[1]), TextInputFormat.class, CanonicalMapper.class);
    MultipleInputs.addInputPath(job, new Path(args[2]), TextInputFormat.class, LibraryThingMapper.class);
    MultipleInputs.addInputPath(job, new Path(args[3]), TextInputFormat.class, LTScrapedMapper.class);

    job.waitForCompletion(true);
}
开发者ID:mouse-reeve，项目名称:book-merger，代码行数:18，代码来源:BookMerger.java

示例7: process

import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; //导入方法依赖的package包/类
@Override
public void process(Annotation annotation, Job job, Object target)
		throws ToolException {
	for (Input input : ((MultiInput)annotation).value()) {
		Path path = getInputAsPath(input.path());
		if (input.mapper() == Mapper.class) {
			MultipleInputs.addInputPath(job, path, input.format());
		}
		else {
			MultipleInputs.addInputPath(job, path, input.format(), input.mapper());
			// Need to call again here so the call is captured by our aspect which
			// will replace it with the annotated delegating mapper class for resource
			// injection if required.
			job.setMapperClass(DelegatingMapper.class);
		}
	}
}
开发者ID:conversant，项目名称:mara，代码行数:18，代码来源:MultiInputAnnotationHandler.java

示例8: run

import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; //导入方法依赖的package包/类
public int run(String[] args) throws Exception {
	Job job=Job.getInstance(getConf(), "reduce side join");
	job.setJarByClass(getClass());
	
	MultipleInputs.addInputPath(job, new Path(args[0]), TextInputFormat.class,ReduceSideJoinMasterMap.class);
	MultipleInputs.addInputPath(job, new Path(args[1]), TextInputFormat.class,ReduceSideJoinMasterMap.class);
	
	job.setMapOutputKeyClass(IntPair.class);
	job.setMapOutputValueClass(Text.class);
	job.setPartitionerClass(ReducesidejoinPartitioner.class);
	job.setGroupingComparatorClass(ReduceSideJoinGroupingComparator.class);
	
	job.setReducerClass(ReduceSideJoinReduce.class);
	job.setOutputKeyClass(IntWritable.class);
	job.setOutputValueClass(Text.class);
	
	job.setOutputFormatClass(TextOutputFormat.class);
	TextOutputFormat.setOutputPath(job, new Path(args[2]));
	
	return job.waitForCompletion(true)?0:1; 
}
开发者ID:makelove，项目名称:book-hadoop-hacks，代码行数:22，代码来源:TestReduceSideJoin.java

示例9: run

import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  Configuration conf = getConf();
  //conf.set("mapreduce.fieldoutput.header", "ct_audit,ct_action");
  Job job = new Job(conf);
  job.setJobName("test fieldInput");
  job.setJarByClass(TestRun.class);
  MultipleInputs.addInputPath(job, new Path(args[0]), FieldInputFormat.class, CTMapper.class);
  job.setNumReduceTasks(0);
  //FileOutputFormat.setOutputPath(job, new Path(args[1]));
  FieldOutputFormat.setOutputPath(job, new Path(args[1]));
  job.setOutputFormatClass(FieldOutputFormat.class);
  job.submit();
  job.waitForCompletion(true);
  return 0;
}
开发者ID:dryman，项目名称:hadoop-fieldformat，代码行数:17，代码来源:TestRun.java

示例10: run

import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {
	Configuration conf = new Configuration();
	Job job = new Job(conf, "PostCommentHeirarchy");
	job.setJarByClass(PostCommentHierarchy.class);

	MultipleInputs.addInputPath(job, new Path(args[0]),
			TextInputFormat.class, PostMapper.class);
	MultipleInputs.addInputPath(job, new Path(args[1]),
			TextInputFormat.class, CommentMapper.class);

	job.setReducerClass(PostCommentHierarchyReducer.class);

	job.setOutputFormatClass(TextOutputFormat.class);
	TextOutputFormat.setOutputPath(job, new Path(args[2]));

	job.setOutputKeyClass(Text.class);
	job.setOutputValueClass(Text.class);

	return job.waitForCompletion(true) ? 0 : 2;
}
开发者ID:geftimov，项目名称:hadoop-map-reduce-patterns，代码行数:22，代码来源:PostCommentHierarchy.java

示例11: main

import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
	Configuration conf = new Configuration();
	Job job = new Job(conf);
	job.setJarByClass(InsDriver.class);
	// job.setMapperClass(PFMapper.class);
	job.setReducerClass(InsReducer.class);
	// job.setNumReduceTasks(0);
	job.setJobName("Participant Adjustment PoC");

	String busDate = args[3].toString();
	job.getConfiguration().set("BUS_DATE", busDate);

	// map-reduce job.
	Path inputPath1 = new Path(args[0]);
	Path inputPath2 = new Path(args[1]);
	Path outputPath = new Path(args[2]);

	MultipleInputs.addInputPath(job, inputPath1, TextInputFormat.class, PFMapper.class);
	MultipleInputs.addInputPath(job, inputPath2, TextInputFormat.class, BRMapper.class);
	FileOutputFormat.setOutputPath(job, outputPath);
	// TODO: Update the output path for the output directory of the
	// map-reduce job.
	// configuration should contain reference to your namenode

	job.setOutputKeyClass(Text.class);
	job.setOutputValueClass(Text.class);

	// Submit the job and wait for it to finish.
	job.waitForCompletion(true);
}
开发者ID:gatripat，项目名称:InsAdjustment，代码行数:31，代码来源:InsDriver.java

示例12: main

import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
	Configuration conf = new Configuration();
	String[] otherArgs = new GenericOptionsParser(conf, args).getRemainingArgs();
	if (otherArgs.length != 3) {
		System.err.println("Usage: Question3 <in> <out>");
		System.exit(3);
	}

	@SuppressWarnings("deprecation")
	Job job1 = new Job(conf, "averageRating");
	@SuppressWarnings("deprecation")
	Job job2 = new Job(conf,"reduceSideJoin");
	job1.setJarByClass(Question3.class); 
	job1.setMapperClass(TopTenMap.class);
	job1.setReducerClass(TopTenReduce.class);

	job1.setOutputKeyClass(Text.class);
	job1.setOutputValueClass(FloatWritable.class);

	FileInputFormat.addInputPath(job1, new Path(otherArgs[0]));    
	FileOutputFormat.setOutputPath(job1, new Path("/bxr140530/Asgn/temp"));

	if(job1.waitForCompletion(true))
	{
		job2.setOutputKeyClass(Text.class);
		job2.setOutputValueClass(Text.class);
		job2.setJarByClass(Question3.class);
		job2.setMapperClass(TopTenJoinMap.class);
		job2.setReducerClass(TopTenJoinReduce.class);

		MultipleInputs.addInputPath(job2,new Path("/bxr140530/Asgn/temp"),TextInputFormat.class,TopTenJoinMap.class);
		MultipleInputs.addInputPath(job2,new Path(otherArgs[1]),TextInputFormat.class,BusinessMap.class);
		FileOutputFormat.setOutputPath(job2, new Path(otherArgs[2]));

		job2.waitForCompletion(true);


	}
}
开发者ID:BhargaviRavula，项目名称:Bigdata，代码行数:40，代码来源:Question3.java

示例13: initJoinMRJob

import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; //导入方法依赖的package包/类
public static void initJoinMRJob(Job job, String prospectsPath, String spoPath, Class<? extends Mapper<CompositeType,TripleCard,?,?>> mapperClass,
    String outPath, String auths) throws AccumuloSecurityException {

  MultipleInputs.addInputPath(job, new Path(prospectsPath), SequenceFileInputFormat.class, mapperClass);
  MultipleInputs.addInputPath(job, new Path(spoPath), SequenceFileInputFormat.class, mapperClass);
  job.setMapOutputKeyClass(CompositeType.class);
  job.setMapOutputValueClass(TripleCard.class);

  SequenceFileOutputFormat.setOutputPath(job, new Path(outPath));
  job.setOutputFormatClass(SequenceFileOutputFormat.class);
  job.setOutputKeyClass(TripleEntry.class);
  job.setOutputValueClass(CardList.class);

}
开发者ID:apache，项目名称:incubator-rya，代码行数:15，代码来源:JoinSelectStatsUtil.java

示例14: run

import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {

    Configuration conf = getConf();
    String outpath = conf.get(OUTPUTPATH);
    
    Job job = new Job(conf, this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
    job.setJarByClass(this.getClass());
    conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
    
    MultipleInputs.addInputPath(job, new Path(PROSPECTSOUT.getAbsolutePath()), 
            SequenceFileInputFormat.class, JoinSelectAggregateMapper.class);
    MultipleInputs.addInputPath(job,new Path(SPOOUT.getAbsolutePath()) , 
            SequenceFileInputFormat.class, JoinSelectAggregateMapper.class);
    job.setMapOutputKeyClass(CompositeType.class);
    job.setMapOutputValueClass(TripleCard.class);

    tempDir = new File(File.createTempFile(outpath, "txt").getParentFile(), System.currentTimeMillis() + "");
    SequenceFileOutputFormat.setOutputPath(job, new Path(tempDir.getAbsolutePath()));
    job.setOutputFormatClass(SequenceFileOutputFormat.class);
    job.setOutputKeyClass(TripleEntry.class);
    job.setOutputValueClass(CardList.class);


    job.setSortComparatorClass(JoinSelectSortComparator.class);
    job.setGroupingComparatorClass(JoinSelectGroupComparator.class);
    job.setPartitionerClass(JoinSelectPartitioner.class);
    job.setReducerClass(JoinReducer.class);
    job.setNumReduceTasks(32);
    job.waitForCompletion(true);
    
    return job.isSuccessful() ? 0 : 1;          
}
开发者ID:apache，项目名称:incubator-rya，代码行数:34，代码来源:JoinSelectStatisticsTest.java

示例15: configureAccumuloInput

import org.apache.hadoop.mapreduce.lib.input.MultipleInputs; //导入方法依赖的package包/类
/**
 * Set up the MapReduce job to use Accumulo as an input.
 * @param tableMapper Mapper class to use
 */
protected void configureAccumuloInput(Class<? extends Mapper<Key,Value,?,?>> tableMapper)
        throws AccumuloSecurityException {
    MRReasoningUtils.configureAccumuloInput(job);
    MultipleInputs.addInputPath(job, new Path("/tmp/input"),
        AccumuloInputFormat.class, tableMapper);
}
开发者ID:apache，项目名称:incubator-rya，代码行数:11，代码来源:AbstractReasoningTool.java


注：本文中的org.apache.hadoop.mapreduce.lib.input.MultipleInputs.addInputPath方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。