Java TableMapReduceUtil类代码示例

本文整理汇总了Java中org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil类的典型用法代码示例。如果您正苦于以下问题：Java TableMapReduceUtil类的具体用法？Java TableMapReduceUtil怎么用？Java TableMapReduceUtil使用的例子？那么, 这里精选的类代码示例或许可以为您提供帮助。

TableMapReduceUtil类属于org.apache.hadoop.hbase.mapreduce包，在下文中一共展示了TableMapReduceUtil类的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: doVerify

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入依赖的package包/类
private Job doVerify(Configuration conf, HTableDescriptor htd, String... auths)
    throws IOException, InterruptedException, ClassNotFoundException {
  Path outputDir = getTestDir(TEST_NAME, "verify-output");
  Job job = new Job(conf);
  job.setJarByClass(this.getClass());
  job.setJobName(TEST_NAME + " Verification for " + htd.getTableName());
  setJobScannerConf(job);
  Scan scan = new Scan();
  scan.setAuthorizations(new Authorizations(auths));
  TableMapReduceUtil.initTableMapperJob(htd.getTableName().getNameAsString(), scan,
      VerifyMapper.class, NullWritable.class, NullWritable.class, job);
  TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
  int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING);
  TableMapReduceUtil.setScannerCaching(job, scannerCaching);
  job.setNumReduceTasks(0);
  FileOutputFormat.setOutputPath(job, outputDir);
  assertTrue(job.waitForCompletion(true));
  return job;
}

开发者ID:fengchen8086，项目名称:ditb，代码行数:20，代码来源:IntegrationTestWithCellVisibilityLoadAndVerify.java

示例2: doLoad

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入依赖的package包/类
protected Job doLoad(Configuration conf, HTableDescriptor htd) throws Exception {
  Path outputDir = getTestDir(TEST_NAME, "load-output");
  LOG.info("Load output dir: " + outputDir);

  NMapInputFormat.setNumMapTasks(conf, conf.getInt(NUM_MAP_TASKS_KEY, NUM_MAP_TASKS_DEFAULT));
  conf.set(TABLE_NAME_KEY, htd.getTableName().getNameAsString());

  Job job = Job.getInstance(conf);
  job.setJobName(TEST_NAME + " Load for " + htd.getTableName());
  job.setJarByClass(this.getClass());
  setMapperClass(job);
  job.setInputFormatClass(NMapInputFormat.class);
  job.setNumReduceTasks(0);
  setJobScannerConf(job);
  FileOutputFormat.setOutputPath(job, outputDir);

  TableMapReduceUtil.addDependencyJars(job);

  TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
  TableMapReduceUtil.initCredentials(job);
  assertTrue(job.waitForCompletion(true));
  return job;
}

开发者ID:fengchen8086，项目名称:ditb，代码行数:24，代码来源:IntegrationTestLoadAndVerify.java

示例3: doVerify

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入依赖的package包/类
protected void doVerify(Configuration conf, HTableDescriptor htd) throws Exception {
  Path outputDir = getTestDir(TEST_NAME, "verify-output");
  LOG.info("Verify output dir: " + outputDir);

  Job job = Job.getInstance(conf);
  job.setJarByClass(this.getClass());
  job.setJobName(TEST_NAME + " Verification for " + htd.getTableName());
  setJobScannerConf(job);

  Scan scan = new Scan();

  TableMapReduceUtil.initTableMapperJob(
      htd.getTableName().getNameAsString(), scan, VerifyMapper.class,
      BytesWritable.class, BytesWritable.class, job);
  TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
  int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING);
  TableMapReduceUtil.setScannerCaching(job, scannerCaching);

  job.setReducerClass(VerifyReducer.class);
  job.setNumReduceTasks(conf.getInt(NUM_REDUCE_TASKS_KEY, NUM_REDUCE_TASKS_DEFAULT));
  FileOutputFormat.setOutputPath(job, outputDir);
  assertTrue(job.waitForCompletion(true));

  long numOutputRecords = job.getCounters().findCounter(Counters.ROWS_WRITTEN).getValue();
  assertEquals(0, numOutputRecords);
}

开发者ID:fengchen8086，项目名称:ditb，代码行数:27，代码来源:IntegrationTestLoadAndVerify.java

示例4: jobSetup

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入依赖的package包/类
@Override
protected void jobSetup(Job job) throws IOException, ImportException {
  super.jobSetup(job);

  // we shouldn't have gotten here if bulk load dir is not set
  // so let's throw a ImportException
  if(getContext().getDestination() == null){
    throw new ImportException("Can't run HBaseBulkImportJob without a " +
        "valid destination directory.");
  }

  TableMapReduceUtil.addDependencyJars(job.getConfiguration(), Preconditions.class);
  FileOutputFormat.setOutputPath(job, getContext().getDestination());
  HTable hTable = new HTable(job.getConfiguration(), options.getHBaseTable());
  HFileOutputFormat.configureIncrementalLoad(job, hTable);
}

开发者ID:aliyun，项目名称:aliyun-maxcompute-data-collectors，代码行数:17，代码来源:HBaseBulkImportJob.java

示例5: doMapReduce

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入依赖的package包/类
private void doMapReduce(final Class<? extends Test> cmd) throws IOException,
      InterruptedException, ClassNotFoundException {
  Configuration conf = getConf();
  Path inputDir = writeInputFile(conf);
  conf.set(EvaluationMapTask.CMD_KEY, cmd.getName());
  conf.set(EvaluationMapTask.PE_KEY, getClass().getName());
  Job job = Job.getInstance(conf);
  job.setJarByClass(PerformanceEvaluation.class);
  job.setJobName("HBase Performance Evaluation");

  job.setInputFormatClass(PeInputFormat.class);
  PeInputFormat.setInputPaths(job, inputDir);

  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(LongWritable.class);

  job.setMapperClass(EvaluationMapTask.class);
  job.setReducerClass(LongSumReducer.class);
  job.setNumReduceTasks(1);

  job.setOutputFormatClass(TextOutputFormat.class);
  TextOutputFormat.setOutputPath(job, new Path(inputDir.getParent(), "outputs"));
  TableMapReduceUtil.addDependencyJars(job);
  TableMapReduceUtil.initCredentials(job);
  job.waitForCompletion(true);
}

开发者ID:fengchen8086，项目名称:ditb，代码行数:27，代码来源:PerformanceEvaluation.java

示例6: configureJob

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入依赖的package包/类
/**
 * Job configuration.
 */
public static Job configureJob(Configuration conf, String [] args)
throws IOException {
  Path inputPath = new Path(args[0]);
  String tableName = args[1];
  Job job = new Job(conf, NAME + "_" + tableName);
  job.setJarByClass(Uploader.class);
  FileInputFormat.setInputPaths(job, inputPath);
  job.setInputFormatClass(SequenceFileInputFormat.class);
  job.setMapperClass(Uploader.class);
  // No reducers.  Just write straight to table.  Call initTableReducerJob
  // because it sets up the TableOutputFormat.
  TableMapReduceUtil.initTableReducerJob(tableName, null, job);
  job.setNumReduceTasks(0);
  return job;
}

开发者ID:fengchen8086，项目名称:ditb，代码行数:19，代码来源:SampleUploader.java

示例7: runTestOnTable

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入依赖的package包/类
private void runTestOnTable() throws InterruptedException, ClassNotFoundException {
    Job job = null;
    try {
        Configuration conf = graph.configuration().toHBaseConfiguration();
        job = Job.getInstance(conf, "test123");
        job.setOutputFormatClass(NullOutputFormat.class);
        job.setNumReduceTasks(0);
        Scan scan = new Scan();
        scan.addColumn(FAMILY_NAME, COLUMN_NAME);
        scan.setTimeRange(MINSTAMP, MAXSTAMP);
        scan.setMaxVersions();
        TableMapReduceUtil.initTableMapperJob(TABLE_NAME.getNameAsString(),
                scan, ProcessTimeRangeMapper.class, Text.class, Text.class, job,
                true, TableInputFormat.class);
        job.waitForCompletion(true);
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    } finally {
        if (job != null) {
            FileUtil.fullyDelete(
                    new File(job.getConfiguration().get("hadoop.tmp.dir")));
        }
    }
}

开发者ID:rayokota，项目名称:hgraphdb，代码行数:26，代码来源:TableInputFormatTest.java

示例8: run

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
    try {
        final Configuration configuration = HBaseConfiguration.create(getConf());
        setConf(configuration);
        final Job job = Job.getInstance(configuration, "phoenix-mr-order_stats-job");
        final String selectQuery = "SELECT ORDER_ID, CUST_ID, AMOUNT FROM ORDERS ";
        // set the input table and select query. you can also pass in the list of columns
        PhoenixMapReduceUtil.setInput(job, OrderWritable.class, "ORDERS", selectQuery);
        // set the output table name and the list of columns.
        PhoenixMapReduceUtil.setOutput(job, "ORDER_STATS", "CUST_ID, AMOUNT");
        job.setMapperClass(OrderMapper.class);
        job.setReducerClass(OrderReducer.class);
        job.setOutputFormatClass(PhoenixOutputFormat.class);
        job.setMapOutputKeyClass(LongWritable.class);
        job.setMapOutputValueClass(DoubleWritable.class);
        job.setOutputKeyClass(NullWritable.class);
        job.setOutputValueClass(OrderWritable.class);
        TableMapReduceUtil.addDependencyJars(job);
        job.waitForCompletion(true);
        return 0;
    } catch (Exception ex) {
        LOG.error(String.format("An exception [%s] occurred while performing the job: ", ex.getMessage()));
        return -1;
    }
}

开发者ID:mravi，项目名称:pro-phoenix，代码行数:27，代码来源:OrderStatsApp.java

示例9: doLoad

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入依赖的package包/类
private void doLoad(Configuration conf, HTableDescriptor htd) throws Exception {
  Path outputDir = getTestDir(TEST_NAME, "load-output");

  NMapInputFormat.setNumMapTasks(conf, conf.getInt(NUM_MAP_TASKS_KEY, NUM_MAP_TASKS_DEFAULT));
  conf.set(TABLE_NAME_KEY, htd.getNameAsString());

  Job job = new Job(conf);
  job.setJobName(TEST_NAME + " Load for " + htd.getNameAsString());
  job.setJarByClass(this.getClass());
  job.setMapperClass(LoadMapper.class);
  job.setInputFormatClass(NMapInputFormat.class);
  job.setNumReduceTasks(0);
  FileOutputFormat.setOutputPath(job, outputDir);

  TableMapReduceUtil.addDependencyJars(job);
  TableMapReduceUtil.addDependencyJars(
      job.getConfiguration(), HTable.class, Lists.class);
  TableMapReduceUtil.initCredentials(job);
  assertTrue(job.waitForCompletion(true));
}

开发者ID:fengchen8086，项目名称:LCIndex-HBase-0.94.16，代码行数:21，代码来源:IntegrationTestLoadAndVerify.java

示例10: doVerify

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入依赖的package包/类
private void doVerify(Configuration conf, HTableDescriptor htd) throws Exception {
  Path outputDir = getTestDir(TEST_NAME, "verify-output");

  Job job = new Job(conf);
  job.setJarByClass(this.getClass());
  job.setJobName(TEST_NAME + " Verification for " + htd.getNameAsString());

  Scan scan = new Scan();

  TableMapReduceUtil.initTableMapperJob(
      htd.getNameAsString(), scan, VerifyMapper.class,
      BytesWritable.class, BytesWritable.class, job);
  int scannerCaching = conf.getInt("verify.scannercaching", SCANNER_CACHING);
  TableMapReduceUtil.setScannerCaching(job, scannerCaching);

  job.setReducerClass(VerifyReducer.class);
  job.setNumReduceTasks(conf.getInt(NUM_REDUCE_TASKS_KEY, NUM_REDUCE_TASKS_DEFAULT));
  FileOutputFormat.setOutputPath(job, outputDir);
  assertTrue(job.waitForCompletion(true));

  long numOutputRecords = job.getCounters().findCounter(Counters.ROWS_WRITTEN).getValue();
  assertEquals(0, numOutputRecords);
}

开发者ID:fengchen8086，项目名称:LCIndex-HBase-0.94.16，代码行数:24，代码来源:IntegrationTestLoadAndVerify.java

示例11: run

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入依赖的package包/类
@Override
public int run(String[] args) throws Exception {
    Configuration conf = this.getConf();
    Job job = Job.getInstance(conf, "loadlogs mr");
    job.setJarByClass(LoadLogsMR.class);
 
    job.setInputFormatClass(TextInputFormat.class);
    
    TableMapReduceUtil.addDependencyJars(job);
    TableMapReduceUtil.initTableReducerJob(args[2], LoadLogsReducer.class, job);
    job.setNumReduceTasks(3);
   
    job.setMapOutputKeyClass(LongWritable.class);
    job.setMapOutputValueClass(Text.class);
    TextInputFormat.addInputPath(job, new Path(args[0]));
    TextInputFormat.addInputPath(job, new Path(args[1]));
    TextOutputFormat.setOutputPath(job, new Path(args[2]));
 
    return job.waitForCompletion(true) ? 0 : 1;
}

开发者ID:hanhanwu，项目名称:Hanhan-HBase-MapReduce-in-Java，代码行数:21，代码来源:LoadLogsMR.java

示例12: process

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入依赖的package包/类
@Override
public void process(Annotation annotation, Job job, Object target)
		throws ToolException {

	TableInput tableInput = (TableInput)annotation;

	// Base setup of the table mapper job
	Configuration conf = job.getConfiguration();
	HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));

	try {
		// Add dependencies
		TableMapReduceUtil.addDependencyJars(job);

		String tableName = getTableName(tableInput);
		Scan scan = getScan(tableInput);

		job.setInputFormatClass(TableInputFormat.class);
		conf.set(TableInputFormat.INPUT_TABLE, tableName);
		conf.set(TableInputFormat.SCAN, convertScanToString(scan));

	} catch (IOException e) {
		throw new ToolException(e);
	}
}

开发者ID:conversant，项目名称:mara，代码行数:26，代码来源:TableInputAnnotationHandler.java

示例13: process

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入依赖的package包/类
@Override
public void process(Annotation annotation, Job job, Object target)
		throws ToolException {

	TableOutput tableOutput = (TableOutput)annotation;

	// Base setup of the table job
	Configuration conf = job.getConfiguration();
	HBaseConfiguration.merge(conf, HBaseConfiguration.create(conf));

	// Add dependencies
	try {
		TableMapReduceUtil.addDependencyJars(job);
	} catch (IOException e) {
		throw new ToolException(e);
	}

	// Set table output format
	job.setOutputFormatClass(TableOutputFormat.class);

	// Set the table name
	String tableName = (String)this.evaluateExpression(tableOutput.value());
	job.getConfiguration().set(TableOutputFormat.OUTPUT_TABLE, tableName);

}

开发者ID:conversant，项目名称:mara，代码行数:26，代码来源:TableOutputAnnotationHandler.java

示例14: doMapReduce

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入依赖的package包/类
private void doMapReduce(final Class<? extends Test> cmd) throws IOException,
      InterruptedException, ClassNotFoundException {
  Configuration conf = getConf();
  Path inputDir = writeInputFile(conf);
  conf.set(EvaluationMapTask.CMD_KEY, cmd.getName());
  conf.set(EvaluationMapTask.PE_KEY, getClass().getName());
  Job job = new Job(conf);
  job.setJarByClass(PerformanceEvaluation.class);
  job.setJobName("HBase Performance Evaluation");

  job.setInputFormatClass(PeInputFormat.class);
  PeInputFormat.setInputPaths(job, inputDir);

  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(LongWritable.class);

  job.setMapperClass(EvaluationMapTask.class);
  job.setReducerClass(LongSumReducer.class);
  job.setNumReduceTasks(1);

  job.setOutputFormatClass(TextOutputFormat.class);
  TextOutputFormat.setOutputPath(job, new Path(inputDir.getParent(), "outputs"));
  TableMapReduceUtil.addDependencyJars(job);
  TableMapReduceUtil.initCredentials(job);
  job.waitForCompletion(true);
}

开发者ID:tenggyut，项目名称:HIndex，代码行数:27，代码来源:PerformanceEvaluation.java

示例15: doLoad

import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; //导入依赖的package包/类
protected Job doLoad(Configuration conf, HTableDescriptor htd) throws Exception {
  Path outputDir = getTestDir(TEST_NAME, "load-output");

  NMapInputFormat.setNumMapTasks(conf, conf.getInt(NUM_MAP_TASKS_KEY, NUM_MAP_TASKS_DEFAULT));
  conf.set(TABLE_NAME_KEY, htd.getTableName().getNameAsString());

  Job job = new Job(conf);
  job.setJobName(TEST_NAME + " Load for " + htd.getTableName());
  job.setJarByClass(this.getClass());
  setMapperClass(job);
  job.setInputFormatClass(NMapInputFormat.class);
  job.setNumReduceTasks(0);
  setJobScannerConf(job);
  FileOutputFormat.setOutputPath(job, outputDir);

  TableMapReduceUtil.addDependencyJars(job);

  TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);
  TableMapReduceUtil.initCredentials(job);
  assertTrue(job.waitForCompletion(true));
  return job;
}

开发者ID:tenggyut，项目名称:HIndex，代码行数:23，代码来源:IntegrationTestLoadAndVerify.java

注：本文中的org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。