当前位置: 首页>>代码示例>>Java>>正文

Java TableMapReduceUtil.initTableReducerJob方法代码示例

本文整理汇总了Java中org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.initTableReducerJob方法的典型用法代码示例。如果您正苦于以下问题：Java TableMapReduceUtil.initTableReducerJob方法的具体用法？Java TableMapReduceUtil.initTableReducerJob怎么用？Java TableMapReduceUtil.initTableReducerJob使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil的用法示例。


在下文中一共展示了TableMapReduceUtil.initTableReducerJob方法的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: configureJob

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入方法依赖的package包/类
/**
 * Job configuration.
 */
public static Job configureJob(Configuration conf, String [] args)
throws IOException {
  Path inputPath = new Path(args[0]);
  String tableName = args[1];
  Job job = new Job(conf, NAME + "_" + tableName);
  job.setJarByClass(Uploader.class);
  FileInputFormat.setInputPaths(job, inputPath);
  job.setInputFormatClass(SequenceFileInputFormat.class);
  job.setMapperClass(Uploader.class);
  // No reducers.  Just write straight to table.  Call initTableReducerJob
  // because it sets up the TableOutputFormat.
  TableMapReduceUtil.initTableReducerJob(tableName, null, job);
  job.setNumReduceTasks(0);
  return job;
}
开发者ID:fengchen8086，项目名称:ditb，代码行数:19，代码来源:SampleUploader.java

示例2: run

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {
    Configuration conf = this.getConf();
    Job job = Job.getInstance(conf, "loadlogs mr");
    job.setJarByClass(LoadLogsMR.class);
 
    job.setInputFormatClass(TextInputFormat.class);
    
    TableMapReduceUtil.addDependencyJars(job);
    TableMapReduceUtil.initTableReducerJob(args[2], LoadLogsReducer.class, job);
    job.setNumReduceTasks(3);
   
    job.setMapOutputKeyClass(LongWritable.class);
    job.setMapOutputValueClass(Text.class);
    TextInputFormat.addInputPath(job, new Path(args[0]));
    TextInputFormat.addInputPath(job, new Path(args[1]));
    TextOutputFormat.setOutputPath(job, new Path(args[2]));
 
    return job.waitForCompletion(true) ? 0 : 1;
}
开发者ID:hanhanwu，项目名称:Hanhan-HBase-MapReduce-in-Java，代码行数:21，代码来源:LoadLogsMR.java

示例3: setupHBaseJob

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入方法依赖的package包/类
public static void setupHBaseJob(Job job, String sourceTable, String cf) throws IOException {
        Scan scan = new Scan();
        if(cf != null) {
            scan.addFamily(Bytes.toBytes(cf));
        }
        scan.setCaching(500);        // 1 is the default in Scan, which will be bad for MapReduce jobs
        scan.setCacheBlocks(false);  // don't set to true for MR jobs
// set other scan attrs

        TableMapReduceUtil.initTableMapperJob(
                sourceTable,      // input table
                scan,	          // Scan instance to control CF and attribute selection
                PrunerMapper.class,   // mapper class
                null,	          // mapper output key
                null,	          // mapper output value
                job);
        TableMapReduceUtil.initTableReducerJob(
                sourceTable,      // output table
                null,             // reducer class
                job);
    }
开发者ID:apache，项目名称:metron，代码行数:22，代码来源:LeastRecentlyUsedPruner.java

示例4: configureJob

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入方法依赖的package包/类
/**
 * Job configuration.
 */
public static Job configureJob(Configuration conf, String[] args) throws IOException {
	Path inputPath = new Path(args[0]);
	String tableName = args[1];
	//Job job = new Job(conf, NAME + "_" + tableName);
	
	Job job = Job.getInstance(conf);
	
	//job.setJarByClass(Uploader.class);
	FileInputFormat.setInputPaths(job, inputPath);
	job.setInputFormatClass(SequenceFileInputFormat.class);
	job.setMapperClass(Uploader.class);
	// No reducers. Just write straight to table. Call initTableReducerJob
	// because it sets up the TableOutputFormat.
	TableMapReduceUtil.initTableReducerJob(tableName, null, job);
	job.setNumReduceTasks(0);
	return job;
}
开发者ID:lhfei，项目名称:hbase-in-action，代码行数:21，代码来源:SampleUploader.java

示例5: run

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入方法依赖的package包/类
public void run() {

		try {
			Job job = Job.getInstance(HBaseContext.config, "UpdateClusterJob");
			job.setJarByClass(UpdateClusterJob.class);

			Scan scan = new Scan();
			scan.setCaching(500);
			scan.setCacheBlocks(false);
			TableMapReduceUtil.initTableMapperJob(
					Constants.hbase_cluster_model_table, scan,
					HBaseReadMapper.class, Text.class, Text.class, job);
			TableMapReduceUtil.initTableReducerJob(
					Constants.hbase_cluster_model_table,
					HBaseWriteReducer.class, job);
			job.setNumReduceTasks(4);

			boolean b = job.waitForCompletion(true);
			if (!b) {
				throw new IOException("error with job!");
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
开发者ID:bytegriffin，项目名称:recsys-offline，代码行数:26，代码来源:UpdateClusterJob.java

示例6: configureJob

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入方法依赖的package包/类
public Job configureJob(Configuration conf, String outputTable) throws IOException {
  conf.setInt("mapreduce.map.failures.maxpercent", 5);
  conf.setInt("mapred.max.map.failures.percent", 5);
  conf.setInt("mapred.max.tracker.failures", 20);

  Job job = Job.getInstance(conf);
  job.setJarByClass(CouchbaseViewToHBaseImporter.class);

  // Input
  job.setInputFormatClass(CouchbaseViewInputFormat.class);

  // Mapper
  job.setMapperClass(CouchbaseViewToHBaseMapper.class);

  // Reducer
  job.setNumReduceTasks(0);

  // Output
  TableMapReduceUtil.initTableReducerJob(outputTable, IdentityTableReducer.class, job);

  return job;
}
开发者ID:Avira，项目名称:couchdoop，代码行数:23，代码来源:CouchbaseViewToHBaseImporter.java

示例7: main

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入方法依赖的package包/类
public static void main(String args[]) throws IOException, ClassNotFoundException, InterruptedException {
	//Configuration hadoopConf = new Configuration();
    Configuration conf = HBaseConfiguration.create();
       conf.set("mapred.child.java.opts", "-Xmx1536m");
	Job job = new Job(conf, "Similarity output HBase insertion job");
	job.getConfiguration().set("familyBytes", args[2]);
	FileInputFormat.addInputPath(job, new Path(args[0]));
	job.setJarByClass(SimilarityInsertJob.class);
	job.setInputFormatClass(TextInputFormat.class);
	job.setMapperClass(SimilarityInsertMapper.class);
	job.setMapOutputKeyClass(Text.class);
	job.setMapOutputValueClass(Text.class);
	TableMapReduceUtil.initTableReducerJob(args[1], SimilarityInsertReducer.class, job);
	job.setNumReduceTasks(2);
	job.waitForCompletion(true);
}
开发者ID:h20，项目名称:MusicBigData，代码行数:17，代码来源:SimilarityInsertJob.java

示例8: prepareMostSimilarItems

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入方法依赖的package包/类
protected Job prepareMostSimilarItems(Path inputFile, String outputTable) throws IOException
{

	Job job = new Job(new Configuration(getConf()));
	job.setJobName(getCustomJobName(job, RecommendationsImportMap.class, RecommendationsImportReduce.class));

	job.getConfiguration().set(TaskConfig.COLLECTION, task.getConfig().get(TaskConfig.COLLECTION));
	job.getConfiguration().set(TaskConfig.RECOMMENDER, task.getConfig().get(TaskConfig.RECOMMENDER));

	job.setMapperClass(RecommendationsImportMap.class);
	job.setInputFormatClass(SequenceFileInputFormat.class);
	job.setMapOutputKeyClass(LongWritable.class);
	job.setMapOutputValueClass(Put.class);
	job.setNumReduceTasks(4);

	TableMapReduceUtil.initTableReducerJob(outputTable, RecommendationsImportReduce.class, job);

	// job.setCombinerClass(RecommendationsImportReduce.class);

	FileInputFormat.addInputPath(job, inputFile);
	return job;
}
开发者ID:beeldengeluid，项目名称:zieook，代码行数:23，代码来源:ItemSimilarityJobZieook.java

示例9: execute

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入方法依赖的package包/类
@Override
public boolean execute() throws Exception
{

	Configuration conf = getConf();

	Job job = new Job(conf);
	job.setJobName(getCustomJobName(job, RatingImportMap.class, RatingImportReduce.class));
	job.setMapperClass(RatingImportMap.class);
	job.setMapOutputKeyClass(LongWritable.class);
	job.setMapOutputValueClass(Put.class);
	job.setNumReduceTasks(4);
	TableMapReduceUtil.initTableReducerJob(getOutputTable(), RatingImportReduce.class, job);
	FileInputFormat.addInputPath(job, inputFile);

	// printConfiguration();
	boolean result = task.setCurrentJob(job).waitForCompletion(LOG.isDebugEnabled());
	LOG.info("import done, result: {}", result);
	if (result)
	{
		cleanup(inputFile);
	}
	return result;
}
开发者ID:beeldengeluid，项目名称:zieook，代码行数:25，代码来源:RatingImportTool.java

示例10: execute

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入方法依赖的package包/类
@Override
public boolean execute() throws Exception
{
	Configuration conf = getConf();
	Job job = new Job(conf);

	job.setJobName("recommendations data import: <" + getOutputTable() + ">");

	job.setMapperClass(RecommendationsImportMap.class);

	job.setMapOutputKeyClass(LongWritable.class);
	job.setMapOutputValueClass(Put.class);

	TableMapReduceUtil.initTableReducerJob(getOutputTable(), RecommendationsImportReduce.class, job);
	FileInputFormat.addInputPath(job, new Path(inputFile));

	return task.setCurrentJob(job).waitForCompletion(LOG.isDebugEnabled());
}
开发者ID:beeldengeluid，项目名称:zieook，代码行数:19，代码来源:RecommendationImportTool.java

示例11: processItemRated

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入方法依赖的package包/类
private boolean processItemRated() throws IOException, InterruptedException, ClassNotFoundException
{
	// Get the top-n (count) of given ratings
	// Ratings topRated(@PathParam("cp") String cp, @PathParam("collection") String collection,
	// @QueryParam("startdate") Long startDate, @QueryParam("enddate") Long endDate, @QueryParam("size") Integer
	// size);

	Configuration conf = getConf();

	Job job = new Job(conf);
	job.setJobName(getCustomJobName(job, RatingsPerCollectionItemMap.class, RatingsPerCollectionItemReduce.class));
	// job.getConfiguration().set(TableInputFormat.INPUT_TABLE, userTable);
	job.getConfiguration().set(RatingsPerCollectionItemReduce.DATE, Long.toString(endDate));
	job.getConfiguration().set(TaskConfig.COLLECTION, collection);

	TableMapReduceUtil.initTableMapperJob(userTable, getRatingsScanner(collection, startDate, endDate),
			RatingsPerCollectionItemMap.class, LongWritable.class, LongWritable.class, job);

	// reducer:
	TableMapReduceUtil.initTableReducerJob(getOutputTable(), RatingsPerCollectionItemReduce.class, job);

	return task.setCurrentJob(job).waitForCompletion(LOG.isDebugEnabled());
}
开发者ID:beeldengeluid，项目名称:zieook，代码行数:24，代码来源:StatisticsTool.java

示例12: processUserRated

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入方法依赖的package包/类
private boolean processUserRated() throws IOException, InterruptedException, ClassNotFoundException
{
	// Get the top-n (count) of given ratings
	// Ratings topRated(@PathParam("cp") String cp, @PathParam("collection") String collection,
	// @QueryParam("startdate") Long startDate, @QueryParam("enddate") Long endDate, @QueryParam("size") Integer
	// size);

	Configuration conf = getConf();

	Job job = new Job(conf);
	job.setJobName(getCustomJobName(job, RatingsPerCPUserMap.class, RatingsPerCPUserReduce.class));
	// job.getConfiguration().set(TableInputFormat.INPUT_TABLE, userTable);
	job.getConfiguration().set(RatingsPerCPUserReduce.DATE, Long.toString(endDate));
	job.getConfiguration().set(TaskConfig.COLLECTION, collection);

	TableMapReduceUtil.initTableMapperJob(userTable, getRatingsScanner(collection, startDate, endDate),
			RatingsPerCPUserMap.class, LongWritable.class, LongWritable.class, job);

	// reducer:
	TableMapReduceUtil.initTableReducerJob(getOutputTable(), RatingsPerCPUserReduce.class, job);

	return task.setCurrentJob(job).waitForCompletion(LOG.isDebugEnabled());
}
开发者ID:beeldengeluid，项目名称:zieook，代码行数:24，代码来源:StatisticsTool.java

示例13: processRecommendedItem

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入方法依赖的package包/类
private boolean processRecommendedItem() throws IOException, InterruptedException, ClassNotFoundException
{
	// Get the top-n most recommended items to user
	// Recommends topRecommended(@PathParam("cp") String cp, @PathParam("recommender") String recommender,
	// @PathParam("user") long user, @QueryParam("size") Integer size);

	// Get the top-n most recommended items to user and return them as collection items
	// CollectionItems topRecommendedAsCollection(@PathParam("cp") String cp, @PathParam("recommender") String
	// recommender,
	// @PathParam("user") long user, @QueryParam("size") Integer size);

	Configuration conf = getConf();
	Job job = new Job(conf);
	job.setJobName(getCustomJobName(job, RecommendedItemToUserMap.class, RecommendedItemToUserReduce.class));
	// job.getConfiguration().set(TableInputFormat.INPUT_TABLE, eventLogTable);
	job.getConfiguration().set(RecommendedItemToUserReduce.DATE, Long.toString(endDate));
	job.getConfiguration().set(TaskConfig.COLLECTION, collection);

	TableMapReduceUtil.initTableMapperJob(eventLogTable, getEventLogRecommendedScanner(startDate, endDate),
			RecommendedItemToUserMap.class, UserCollectionItem.class, LongWritable.class, job);

	// reducer:
	TableMapReduceUtil.initTableReducerJob(getOutputTable(), RecommendedItemToUserReduce.class, job);

	return task.setCurrentJob(job).waitForCompletion(LOG.isDebugEnabled());
}
开发者ID:beeldengeluid，项目名称:zieook，代码行数:27，代码来源:StatisticsTool.java

示例14: processRecommended

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入方法依赖的package包/类
private boolean processRecommended() throws IOException, InterruptedException, ClassNotFoundException
{
	Configuration conf = getConf();
	Job job = new Job(conf);
	job.setJobName(getCustomJobName(job, RecommendedToUserMap.class, RecommendedToUserReduce.class));
	// job.getConfiguration().set(TableInputFormat.INPUT_TABLE, eventLogTable);
	job.getConfiguration().set(RecommendedItemToUserReduce.DATE, Long.toString(endDate));
	job.getConfiguration().set(TaskConfig.COLLECTION, collection);

	TableMapReduceUtil.initTableMapperJob(eventLogTable, getEventLogRecommendedScanner(startDate, endDate),
			RecommendedToUserMap.class, UserCollection.class, LongWritable.class, job);

	// reducer:
	TableMapReduceUtil.initTableReducerJob(getOutputTable(), RecommendedToUserReduce.class, job);

	return task.setCurrentJob(job).waitForCompletion(LOG.isDebugEnabled());
}
开发者ID:beeldengeluid，项目名称:zieook，代码行数:18，代码来源:StatisticsTool.java

示例15: run

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {
	Configuration conf = this.getConf();
	this.processArgs(conf, args);

	Job job = Job.getInstance(conf, "analyser_logdata");

	// 设置本地提交job，集群运行，需要代码
	// File jarFile = EJob.createTempJar("target/classes");
	// ((JobConf) job.getConfiguration()).setJar(jarFile.toString());
	// 设置本地提交job，集群运行，需要代码结束

	job.setJarByClass(AnalyserLogDataRunner.class);
	job.setMapperClass(AnalyserLogDataMapper.class);
	job.setMapOutputKeyClass(NullWritable.class);
	job.setMapOutputValueClass(Put.class);
	// 设置reducer配置
	// 1. 集群上运行，打成jar运行(要求addDependencyJars参数为true，默认就是true)
	// TableMapReduceUtil.initTableReducerJob(EventLogConstants.HBASE_NAME_EVENT_LOGS,
	// null, job);
	// 2. 本地运行，要求参数addDependencyJars为false
	TableMapReduceUtil.initTableReducerJob(EventLogConstants.HBASE_NAME_EVENT_LOGS, null, job, null, null, null,
			null, false);
	job.setNumReduceTasks(0);

	// 设置输入路径
	this.setJobInputPaths(job);
	return job.waitForCompletion(true) ? 0 : -1;
}
开发者ID:liuhaozzu，项目名称:big_data，代码行数:30，代码来源:AnalyserLogDataRunner.java


注：本文中的org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.initTableReducerJob方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。