當前位置: 首頁>>代碼示例>>Java>>正文

Java TableMapReduceUtil.initTableReducerJob方法代碼示例

本文整理匯總了Java中org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.initTableReducerJob方法的典型用法代碼示例。如果您正苦於以下問題：Java TableMapReduceUtil.initTableReducerJob方法的具體用法？Java TableMapReduceUtil.initTableReducerJob怎麽用？Java TableMapReduceUtil.initTableReducerJob使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil的用法示例。


在下文中一共展示了TableMapReduceUtil.initTableReducerJob方法的15個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Java代碼示例。

示例1: configureJob

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //導入方法依賴的package包/類
/**
 * Job configuration.
 */
public static Job configureJob(Configuration conf, String [] args)
throws IOException {
  Path inputPath = new Path(args[0]);
  String tableName = args[1];
  Job job = new Job(conf, NAME + "_" + tableName);
  job.setJarByClass(Uploader.class);
  FileInputFormat.setInputPaths(job, inputPath);
  job.setInputFormatClass(SequenceFileInputFormat.class);
  job.setMapperClass(Uploader.class);
  // No reducers.  Just write straight to table.  Call initTableReducerJob
  // because it sets up the TableOutputFormat.
  TableMapReduceUtil.initTableReducerJob(tableName, null, job);
  job.setNumReduceTasks(0);
  return job;
}
開發者ID:fengchen8086，項目名稱:ditb，代碼行數:19，代碼來源:SampleUploader.java

示例2: run

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //導入方法依賴的package包/類
@Override
public int run(String[] args) throws Exception {
    Configuration conf = this.getConf();
    Job job = Job.getInstance(conf, "loadlogs mr");
    job.setJarByClass(LoadLogsMR.class);
 
    job.setInputFormatClass(TextInputFormat.class);
    
    TableMapReduceUtil.addDependencyJars(job);
    TableMapReduceUtil.initTableReducerJob(args[2], LoadLogsReducer.class, job);
    job.setNumReduceTasks(3);
   
    job.setMapOutputKeyClass(LongWritable.class);
    job.setMapOutputValueClass(Text.class);
    TextInputFormat.addInputPath(job, new Path(args[0]));
    TextInputFormat.addInputPath(job, new Path(args[1]));
    TextOutputFormat.setOutputPath(job, new Path(args[2]));
 
    return job.waitForCompletion(true) ? 0 : 1;
}
開發者ID:hanhanwu，項目名稱:Hanhan-HBase-MapReduce-in-Java，代碼行數:21，代碼來源:LoadLogsMR.java

示例3: setupHBaseJob

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //導入方法依賴的package包/類
public static void setupHBaseJob(Job job, String sourceTable, String cf) throws IOException {
        Scan scan = new Scan();
        if(cf != null) {
            scan.addFamily(Bytes.toBytes(cf));
        }
        scan.setCaching(500);        // 1 is the default in Scan, which will be bad for MapReduce jobs
        scan.setCacheBlocks(false);  // don't set to true for MR jobs
// set other scan attrs

        TableMapReduceUtil.initTableMapperJob(
                sourceTable,      // input table
                scan,	          // Scan instance to control CF and attribute selection
                PrunerMapper.class,   // mapper class
                null,	          // mapper output key
                null,	          // mapper output value
                job);
        TableMapReduceUtil.initTableReducerJob(
                sourceTable,      // output table
                null,             // reducer class
                job);
    }
開發者ID:apache，項目名稱:metron，代碼行數:22，代碼來源:LeastRecentlyUsedPruner.java

示例4: configureJob

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //導入方法依賴的package包/類
/**
 * Job configuration.
 */
public static Job configureJob(Configuration conf, String[] args) throws IOException {
	Path inputPath = new Path(args[0]);
	String tableName = args[1];
	//Job job = new Job(conf, NAME + "_" + tableName);
	
	Job job = Job.getInstance(conf);
	
	//job.setJarByClass(Uploader.class);
	FileInputFormat.setInputPaths(job, inputPath);
	job.setInputFormatClass(SequenceFileInputFormat.class);
	job.setMapperClass(Uploader.class);
	// No reducers. Just write straight to table. Call initTableReducerJob
	// because it sets up the TableOutputFormat.
	TableMapReduceUtil.initTableReducerJob(tableName, null, job);
	job.setNumReduceTasks(0);
	return job;
}
開發者ID:lhfei，項目名稱:hbase-in-action，代碼行數:21，代碼來源:SampleUploader.java

示例5: run

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //導入方法依賴的package包/類
public void run() {

		try {
			Job job = Job.getInstance(HBaseContext.config, "UpdateClusterJob");
			job.setJarByClass(UpdateClusterJob.class);

			Scan scan = new Scan();
			scan.setCaching(500);
			scan.setCacheBlocks(false);
			TableMapReduceUtil.initTableMapperJob(
					Constants.hbase_cluster_model_table, scan,
					HBaseReadMapper.class, Text.class, Text.class, job);
			TableMapReduceUtil.initTableReducerJob(
					Constants.hbase_cluster_model_table,
					HBaseWriteReducer.class, job);
			job.setNumReduceTasks(4);

			boolean b = job.waitForCompletion(true);
			if (!b) {
				throw new IOException("error with job!");
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
開發者ID:bytegriffin，項目名稱:recsys-offline，代碼行數:26，代碼來源:UpdateClusterJob.java

示例6: configureJob

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //導入方法依賴的package包/類
public Job configureJob(Configuration conf, String outputTable) throws IOException {
  conf.setInt("mapreduce.map.failures.maxpercent", 5);
  conf.setInt("mapred.max.map.failures.percent", 5);
  conf.setInt("mapred.max.tracker.failures", 20);

  Job job = Job.getInstance(conf);
  job.setJarByClass(CouchbaseViewToHBaseImporter.class);

  // Input
  job.setInputFormatClass(CouchbaseViewInputFormat.class);

  // Mapper
  job.setMapperClass(CouchbaseViewToHBaseMapper.class);

  // Reducer
  job.setNumReduceTasks(0);

  // Output
  TableMapReduceUtil.initTableReducerJob(outputTable, IdentityTableReducer.class, job);

  return job;
}
開發者ID:Avira，項目名稱:couchdoop，代碼行數:23，代碼來源:CouchbaseViewToHBaseImporter.java

示例7: main

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //導入方法依賴的package包/類
public static void main(String args[]) throws IOException, ClassNotFoundException, InterruptedException {
	//Configuration hadoopConf = new Configuration();
    Configuration conf = HBaseConfiguration.create();
       conf.set("mapred.child.java.opts", "-Xmx1536m");
	Job job = new Job(conf, "Similarity output HBase insertion job");
	job.getConfiguration().set("familyBytes", args[2]);
	FileInputFormat.addInputPath(job, new Path(args[0]));
	job.setJarByClass(SimilarityInsertJob.class);
	job.setInputFormatClass(TextInputFormat.class);
	job.setMapperClass(SimilarityInsertMapper.class);
	job.setMapOutputKeyClass(Text.class);
	job.setMapOutputValueClass(Text.class);
	TableMapReduceUtil.initTableReducerJob(args[1], SimilarityInsertReducer.class, job);
	job.setNumReduceTasks(2);
	job.waitForCompletion(true);
}
開發者ID:h20，項目名稱:MusicBigData，代碼行數:17，代碼來源:SimilarityInsertJob.java

示例8: prepareMostSimilarItems

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //導入方法依賴的package包/類
protected Job prepareMostSimilarItems(Path inputFile, String outputTable) throws IOException
{

	Job job = new Job(new Configuration(getConf()));
	job.setJobName(getCustomJobName(job, RecommendationsImportMap.class, RecommendationsImportReduce.class));

	job.getConfiguration().set(TaskConfig.COLLECTION, task.getConfig().get(TaskConfig.COLLECTION));
	job.getConfiguration().set(TaskConfig.RECOMMENDER, task.getConfig().get(TaskConfig.RECOMMENDER));

	job.setMapperClass(RecommendationsImportMap.class);
	job.setInputFormatClass(SequenceFileInputFormat.class);
	job.setMapOutputKeyClass(LongWritable.class);
	job.setMapOutputValueClass(Put.class);
	job.setNumReduceTasks(4);

	TableMapReduceUtil.initTableReducerJob(outputTable, RecommendationsImportReduce.class, job);

	// job.setCombinerClass(RecommendationsImportReduce.class);

	FileInputFormat.addInputPath(job, inputFile);
	return job;
}
開發者ID:beeldengeluid，項目名稱:zieook，代碼行數:23，代碼來源:ItemSimilarityJobZieook.java

示例9: execute

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //導入方法依賴的package包/類
@Override
public boolean execute() throws Exception
{

	Configuration conf = getConf();

	Job job = new Job(conf);
	job.setJobName(getCustomJobName(job, RatingImportMap.class, RatingImportReduce.class));
	job.setMapperClass(RatingImportMap.class);
	job.setMapOutputKeyClass(LongWritable.class);
	job.setMapOutputValueClass(Put.class);
	job.setNumReduceTasks(4);
	TableMapReduceUtil.initTableReducerJob(getOutputTable(), RatingImportReduce.class, job);
	FileInputFormat.addInputPath(job, inputFile);

	// printConfiguration();
	boolean result = task.setCurrentJob(job).waitForCompletion(LOG.isDebugEnabled());
	LOG.info("import done, result: {}", result);
	if (result)
	{
		cleanup(inputFile);
	}
	return result;
}
開發者ID:beeldengeluid，項目名稱:zieook，代碼行數:25，代碼來源:RatingImportTool.java

示例10: execute

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //導入方法依賴的package包/類
@Override
public boolean execute() throws Exception
{
	Configuration conf = getConf();
	Job job = new Job(conf);

	job.setJobName("recommendations data import: <" + getOutputTable() + ">");

	job.setMapperClass(RecommendationsImportMap.class);

	job.setMapOutputKeyClass(LongWritable.class);
	job.setMapOutputValueClass(Put.class);

	TableMapReduceUtil.initTableReducerJob(getOutputTable(), RecommendationsImportReduce.class, job);
	FileInputFormat.addInputPath(job, new Path(inputFile));

	return task.setCurrentJob(job).waitForCompletion(LOG.isDebugEnabled());
}
開發者ID:beeldengeluid，項目名稱:zieook，代碼行數:19，代碼來源:RecommendationImportTool.java

示例11: processItemRated

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //導入方法依賴的package包/類
private boolean processItemRated() throws IOException, InterruptedException, ClassNotFoundException
{
	// Get the top-n (count) of given ratings
	// Ratings topRated(@PathParam("cp") String cp, @PathParam("collection") String collection,
	// @QueryParam("startdate") Long startDate, @QueryParam("enddate") Long endDate, @QueryParam("size") Integer
	// size);

	Configuration conf = getConf();

	Job job = new Job(conf);
	job.setJobName(getCustomJobName(job, RatingsPerCollectionItemMap.class, RatingsPerCollectionItemReduce.class));
	// job.getConfiguration().set(TableInputFormat.INPUT_TABLE, userTable);
	job.getConfiguration().set(RatingsPerCollectionItemReduce.DATE, Long.toString(endDate));
	job.getConfiguration().set(TaskConfig.COLLECTION, collection);

	TableMapReduceUtil.initTableMapperJob(userTable, getRatingsScanner(collection, startDate, endDate),
			RatingsPerCollectionItemMap.class, LongWritable.class, LongWritable.class, job);

	// reducer:
	TableMapReduceUtil.initTableReducerJob(getOutputTable(), RatingsPerCollectionItemReduce.class, job);

	return task.setCurrentJob(job).waitForCompletion(LOG.isDebugEnabled());
}
開發者ID:beeldengeluid，項目名稱:zieook，代碼行數:24，代碼來源:StatisticsTool.java

示例12: processUserRated

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //導入方法依賴的package包/類
private boolean processUserRated() throws IOException, InterruptedException, ClassNotFoundException
{
	// Get the top-n (count) of given ratings
	// Ratings topRated(@PathParam("cp") String cp, @PathParam("collection") String collection,
	// @QueryParam("startdate") Long startDate, @QueryParam("enddate") Long endDate, @QueryParam("size") Integer
	// size);

	Configuration conf = getConf();

	Job job = new Job(conf);
	job.setJobName(getCustomJobName(job, RatingsPerCPUserMap.class, RatingsPerCPUserReduce.class));
	// job.getConfiguration().set(TableInputFormat.INPUT_TABLE, userTable);
	job.getConfiguration().set(RatingsPerCPUserReduce.DATE, Long.toString(endDate));
	job.getConfiguration().set(TaskConfig.COLLECTION, collection);

	TableMapReduceUtil.initTableMapperJob(userTable, getRatingsScanner(collection, startDate, endDate),
			RatingsPerCPUserMap.class, LongWritable.class, LongWritable.class, job);

	// reducer:
	TableMapReduceUtil.initTableReducerJob(getOutputTable(), RatingsPerCPUserReduce.class, job);

	return task.setCurrentJob(job).waitForCompletion(LOG.isDebugEnabled());
}
開發者ID:beeldengeluid，項目名稱:zieook，代碼行數:24，代碼來源:StatisticsTool.java

示例13: processRecommendedItem

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //導入方法依賴的package包/類
private boolean processRecommendedItem() throws IOException, InterruptedException, ClassNotFoundException
{
	// Get the top-n most recommended items to user
	// Recommends topRecommended(@PathParam("cp") String cp, @PathParam("recommender") String recommender,
	// @PathParam("user") long user, @QueryParam("size") Integer size);

	// Get the top-n most recommended items to user and return them as collection items
	// CollectionItems topRecommendedAsCollection(@PathParam("cp") String cp, @PathParam("recommender") String
	// recommender,
	// @PathParam("user") long user, @QueryParam("size") Integer size);

	Configuration conf = getConf();
	Job job = new Job(conf);
	job.setJobName(getCustomJobName(job, RecommendedItemToUserMap.class, RecommendedItemToUserReduce.class));
	// job.getConfiguration().set(TableInputFormat.INPUT_TABLE, eventLogTable);
	job.getConfiguration().set(RecommendedItemToUserReduce.DATE, Long.toString(endDate));
	job.getConfiguration().set(TaskConfig.COLLECTION, collection);

	TableMapReduceUtil.initTableMapperJob(eventLogTable, getEventLogRecommendedScanner(startDate, endDate),
			RecommendedItemToUserMap.class, UserCollectionItem.class, LongWritable.class, job);

	// reducer:
	TableMapReduceUtil.initTableReducerJob(getOutputTable(), RecommendedItemToUserReduce.class, job);

	return task.setCurrentJob(job).waitForCompletion(LOG.isDebugEnabled());
}
開發者ID:beeldengeluid，項目名稱:zieook，代碼行數:27，代碼來源:StatisticsTool.java

示例14: processRecommended

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //導入方法依賴的package包/類
private boolean processRecommended() throws IOException, InterruptedException, ClassNotFoundException
{
	Configuration conf = getConf();
	Job job = new Job(conf);
	job.setJobName(getCustomJobName(job, RecommendedToUserMap.class, RecommendedToUserReduce.class));
	// job.getConfiguration().set(TableInputFormat.INPUT_TABLE, eventLogTable);
	job.getConfiguration().set(RecommendedItemToUserReduce.DATE, Long.toString(endDate));
	job.getConfiguration().set(TaskConfig.COLLECTION, collection);

	TableMapReduceUtil.initTableMapperJob(eventLogTable, getEventLogRecommendedScanner(startDate, endDate),
			RecommendedToUserMap.class, UserCollection.class, LongWritable.class, job);

	// reducer:
	TableMapReduceUtil.initTableReducerJob(getOutputTable(), RecommendedToUserReduce.class, job);

	return task.setCurrentJob(job).waitForCompletion(LOG.isDebugEnabled());
}
開發者ID:beeldengeluid，項目名稱:zieook，代碼行數:18，代碼來源:StatisticsTool.java

示例15: run

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //導入方法依賴的package包/類
@Override
public int run(String[] args) throws Exception {
	Configuration conf = this.getConf();
	this.processArgs(conf, args);

	Job job = Job.getInstance(conf, "analyser_logdata");

	// 設置本地提交job，集群運行，需要代碼
	// File jarFile = EJob.createTempJar("target/classes");
	// ((JobConf) job.getConfiguration()).setJar(jarFile.toString());
	// 設置本地提交job，集群運行，需要代碼結束

	job.setJarByClass(AnalyserLogDataRunner.class);
	job.setMapperClass(AnalyserLogDataMapper.class);
	job.setMapOutputKeyClass(NullWritable.class);
	job.setMapOutputValueClass(Put.class);
	// 設置reducer配置
	// 1. 集群上運行，打成jar運行(要求addDependencyJars參數為true，默認就是true)
	// TableMapReduceUtil.initTableReducerJob(EventLogConstants.HBASE_NAME_EVENT_LOGS,
	// null, job);
	// 2. 本地運行，要求參數addDependencyJars為false
	TableMapReduceUtil.initTableReducerJob(EventLogConstants.HBASE_NAME_EVENT_LOGS, null, job, null, null, null,
			null, false);
	job.setNumReduceTasks(0);

	// 設置輸入路徑
	this.setJobInputPaths(job);
	return job.waitForCompletion(true) ? 0 : -1;
}
開發者ID:liuhaozzu，項目名稱:big_data，代碼行數:30，代碼來源:AnalyserLogDataRunner.java


注：本文中的org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil.initTableReducerJob方法示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台，相關代碼片段篩選自各路編程大神貢獻的開源項目，源碼版權歸原作者所有，傳播和使用請參考對應項目的License；未經允許，請勿轉載。