當前位置: 首頁>>代碼示例>>Java>>正文

Java TextOutputFormat.setOutputPath方法代碼示例

本文整理匯總了Java中org.apache.hadoop.mapreduce.lib.output.TextOutputFormat.setOutputPath方法的典型用法代碼示例。如果您正苦於以下問題：Java TextOutputFormat.setOutputPath方法的具體用法？Java TextOutputFormat.setOutputPath怎麽用？Java TextOutputFormat.setOutputPath使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類org.apache.hadoop.mapreduce.lib.output.TextOutputFormat的用法示例。


在下文中一共展示了TextOutputFormat.setOutputPath方法的15個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Java代碼示例。

示例1: main

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; //導入方法依賴的package包/類
public static void main(String[] args) throws Exception {
	Configuration conf = new Configuration();

	Job job = Job.getInstance(conf);
	job.setJarByClass(Multiplication.class);

	ChainMapper.addMapper(job, CooccurrenceMapper.class, LongWritable.class, Text.class, Text.class, Text.class, conf);
	ChainMapper.addMapper(job, RatingMapper.class, Text.class, Text.class, Text.class, Text.class, conf);

	job.setMapperClass(CooccurrenceMapper.class);
	job.setMapperClass(RatingMapper.class);

	job.setReducerClass(MultiplicationReducer.class);

	job.setMapOutputKeyClass(Text.class);
	job.setMapOutputValueClass(Text.class);
	job.setOutputKeyClass(Text.class);
	job.setOutputValueClass(DoubleWritable.class);

	MultipleInputs.addInputPath(job, new Path(args[0]), TextInputFormat.class, CooccurrenceMapper.class);
	MultipleInputs.addInputPath(job, new Path(args[1]), TextInputFormat.class, RatingMapper.class);

	TextOutputFormat.setOutputPath(job, new Path(args[2]));
	
	job.waitForCompletion(true);
}
開發者ID:yogykwan，項目名稱:mapreduce-samples，代碼行數:27，代碼來源:Multiplication.java

示例2: main

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; //導入方法依賴的package包/類
public static void main(String[] args) throws Exception {

        Configuration conf = new Configuration();

        Job job = Job.getInstance(conf);
        job.setMapperClass(SumMapper.class);
        job.setReducerClass(SumReducer.class);

        job.setJarByClass(Sum.class);

        job.setInputFormatClass(TextInputFormat.class);
        job.setOutputFormatClass(TextOutputFormat.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(DoubleWritable.class);

        TextInputFormat.setInputPaths(job, new Path(args[0]));
        TextOutputFormat.setOutputPath(job, new Path(args[1]));

        job.waitForCompletion(true);
    }
開發者ID:yogykwan，項目名稱:mapreduce-samples，代碼行數:21，代碼來源:Sum.java

示例3: main

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; //導入方法依賴的package包/類
public static void main(String[] args) throws Exception {

		Configuration conf = new Configuration();

		Job job = Job.getInstance(conf);
		job.setMapperClass(DataDividerMapper.class);
		job.setReducerClass(DataDividerReducer.class);

		job.setJarByClass(DataDividerByUser.class);

		job.setInputFormatClass(TextInputFormat.class);
		job.setOutputFormatClass(TextOutputFormat.class);
		job.setOutputKeyClass(IntWritable.class);
		job.setOutputValueClass(Text.class);

		TextInputFormat.setInputPaths(job, new Path(args[0]));
		TextOutputFormat.setOutputPath(job, new Path(args[1]));

		job.waitForCompletion(true);
	}
開發者ID:yogykwan，項目名稱:mapreduce-samples，代碼行數:21，代碼來源:DataDividerByUser.java

示例4: main

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; //導入方法依賴的package包/類
public static void main(String[] args) throws Exception {

        Configuration conf = new Configuration();

        Job job = Job.getInstance(conf);
        job.setMapperClass(NormalizeMapper.class);
        job.setReducerClass(NormalizeReducer.class);

        job.setJarByClass(Normalize.class);

        job.setInputFormatClass(TextInputFormat.class);
        job.setOutputFormatClass(TextOutputFormat.class);
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(Text.class);

        TextInputFormat.setInputPaths(job, new Path(args[0]));
        TextOutputFormat.setOutputPath(job, new Path(args[1]));

        job.waitForCompletion(true);
    }
開發者ID:yogykwan，項目名稱:mapreduce-samples，代碼行數:21，代碼來源:Normalize.java

示例5: doMapReduce

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; //導入方法依賴的package包/類
private void doMapReduce(final Class<? extends Test> cmd) throws IOException,
      InterruptedException, ClassNotFoundException {
  Configuration conf = getConf();
  Path inputDir = writeInputFile(conf);
  conf.set(EvaluationMapTask.CMD_KEY, cmd.getName());
  conf.set(EvaluationMapTask.PE_KEY, getClass().getName());
  Job job = Job.getInstance(conf);
  job.setJarByClass(PerformanceEvaluation.class);
  job.setJobName("HBase Performance Evaluation");

  job.setInputFormatClass(PeInputFormat.class);
  PeInputFormat.setInputPaths(job, inputDir);

  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(LongWritable.class);

  job.setMapperClass(EvaluationMapTask.class);
  job.setReducerClass(LongSumReducer.class);
  job.setNumReduceTasks(1);

  job.setOutputFormatClass(TextOutputFormat.class);
  TextOutputFormat.setOutputPath(job, new Path(inputDir.getParent(), "outputs"));
  TableMapReduceUtil.addDependencyJars(job);
  TableMapReduceUtil.initCredentials(job);
  job.waitForCompletion(true);
}
開發者ID:fengchen8086，項目名稱:ditb，代碼行數:27，代碼來源:PerformanceEvaluation.java

示例6: main

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; //導入方法依賴的package包/類
public static void main(String[] args) throws Exception {
	if (args.length < 2) {
		System.err.println("Usage: WordCount <input path> <result path>");
		return;
	}
	
	final String inputPath = args[0];
	final String outputPath = args[1];
	
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	
	// Set up the Hadoop Input Format
	Job job = Job.getInstance();
	HadoopInputFormat<LongWritable, Text> hadoopInputFormat = new HadoopInputFormat<LongWritable, Text>(new TextInputFormat(), LongWritable.class, Text.class, job);
	TextInputFormat.addInputPath(job, new Path(inputPath));
	
	// Create a Flink job with it
	DataSet<Tuple2<LongWritable, Text>> text = env.createInput(hadoopInputFormat);
	
	// Tokenize the line and convert from Writable "Text" to String for better handling
	DataSet<Tuple2<String, Integer>> words = text.flatMap(new Tokenizer());
	
	// Sum up the words
	DataSet<Tuple2<String, Integer>> result = words.groupBy(0).aggregate(Aggregations.SUM, 1);
	
	// Convert String back to Writable "Text" for use with Hadoop Output Format
	DataSet<Tuple2<Text, IntWritable>> hadoopResult = result.map(new HadoopDatatypeMapper());
	
	// Set up Hadoop Output Format
	HadoopOutputFormat<Text, IntWritable> hadoopOutputFormat = new HadoopOutputFormat<Text, IntWritable>(new TextOutputFormat<Text, IntWritable>(), job);
	hadoopOutputFormat.getConfiguration().set("mapreduce.output.textoutputformat.separator", " ");
	hadoopOutputFormat.getConfiguration().set("mapred.textoutputformat.separator", " "); // set the value for both, since this test
	TextOutputFormat.setOutputPath(job, new Path(outputPath));
	
	// Output & Execute
	hadoopResult.output(hadoopOutputFormat);
	env.execute("Word Count");
}
開發者ID:axbaretto，項目名稱:flink，代碼行數:39，代碼來源:WordCount.java

示例7: run

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; //導入方法依賴的package包/類
@Override
public int run(String[] args) throws Exception {
    Configuration conf = this.getConf();
    Job job = Job.getInstance(conf, "reddit average");
    job.setJarByClass(RedditAverage.class);
 
    job.setInputFormatClass(TextInputFormat.class);
 
    job.setMapperClass(RedditMapper.class);
    job.setCombinerClass(RedditCombiner.class);
    job.setReducerClass(RedditReducer.class);
 
    job.setMapOutputValueClass(LongPairWritable.class);
    
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(DoubleWritable.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    TextInputFormat.addInputPath(job, new Path(args[0]));
    TextInputFormat.addInputPath(job, new Path(args[1]));
    TextOutputFormat.setOutputPath(job, new Path(args[2]));
 
    return job.waitForCompletion(true) ? 0 : 1;
}
開發者ID:hanhanwu，項目名稱:Hanhan-Hadoop-MapReduce，代碼行數:24，代碼來源:RedditAverage.java

示例8: run

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; //導入方法依賴的package包/類
@Override
public int run(String[] args) throws Exception {
    Configuration conf = this.getConf();
    Job job = Job.getInstance(conf, "loadlogs mr");
    job.setJarByClass(LoadLogsMR.class);
 
    job.setInputFormatClass(TextInputFormat.class);
    
    TableMapReduceUtil.addDependencyJars(job);
    TableMapReduceUtil.initTableReducerJob(args[2], LoadLogsReducer.class, job);
    job.setNumReduceTasks(3);
   
    job.setMapOutputKeyClass(LongWritable.class);
    job.setMapOutputValueClass(Text.class);
    TextInputFormat.addInputPath(job, new Path(args[0]));
    TextInputFormat.addInputPath(job, new Path(args[1]));
    TextOutputFormat.setOutputPath(job, new Path(args[2]));
 
    return job.waitForCompletion(true) ? 0 : 1;
}
開發者ID:hanhanwu，項目名稱:Hanhan-HBase-MapReduce-in-Java，代碼行數:21，代碼來源:LoadLogsMR.java

示例9: createAndSubmitJob

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; //導入方法依賴的package包/類
public boolean createAndSubmitJob() throws IOException, ClassNotFoundException, InterruptedException {
    Job job = Job.getInstance(yarnUnit.getConfig());
    job.setJobName(this.getClass().getSimpleName() + "-job");

    job.setNumReduceTasks(1);

    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    job.setMapperClass(CountMapReduce.CountMapper.class);
    job.setReducerClass(CountMapReduce.CountReducer.class);

    job.setInputFormatClass(TextInputFormat.class);
    job.setOutputFormatClass(TextOutputFormat.class);

    TextInputFormat.addInputPath(job, new Path(inputPath));
    TextOutputFormat.setOutputPath(job, new Path(outputPath));

    job.setSpeculativeExecution(false);
    job.setMaxMapAttempts(1);
    return job.waitForCompletion(true);
}
開發者ID:intropro，項目名稱:prairie，代碼行數:23，代碼來源:YarnUnitTest.java

示例10: doVerify

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; //導入方法依賴的package包/類
private int doVerify(Path outputDir, int numReducers) throws IOException, InterruptedException,
    ClassNotFoundException {
  job = new Job(getConf());

  job.setJobName("Link Verifier");
  job.setNumReduceTasks(numReducers);
  job.setJarByClass(getClass());

  setJobScannerConf(job);

  Scan scan = new Scan();
  scan.addColumn(FAMILY_NAME, COLUMN_PREV);
  scan.setCaching(10000);
  scan.setCacheBlocks(false);
  String[] split = labels.split(COMMA);

  scan.setAuthorizations(new Authorizations(split[this.labelIndex * 2],
      split[(this.labelIndex * 2) + 1]));

  TableMapReduceUtil.initTableMapperJob(tableName.getName(), scan, VerifyMapper.class,
      BytesWritable.class, BytesWritable.class, job);
  TableMapReduceUtil.addDependencyJars(job.getConfiguration(), AbstractHBaseTool.class);

  job.getConfiguration().setBoolean("mapreduce.map.speculative", false);

  job.setReducerClass(VerifyReducer.class);
  job.setOutputFormatClass(TextOutputFormat.class);
  TextOutputFormat.setOutputPath(job, outputDir);
  boolean success = job.waitForCompletion(true);

  return success ? 0 : 1;
}
開發者ID:fengchen8086，項目名稱:ditb，代碼行數:33，代碼來源:IntegrationTestBigLinkedListWithVisibility.java

示例11: run

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; //導入方法依賴的package包/類
public int run(String[] args) throws Exception {

    GfxdDataSerializable.initTypes();
    Configuration conf = getConf();

    Path outputPath = new Path(args[0]);
    String hdfsHomeDir = args[1];
    String tableName = args[2];

    outputPath.getFileSystem(conf).delete(outputPath, true);

    conf.set(RowInputFormat.HOME_DIR, hdfsHomeDir);
    conf.set(RowInputFormat.INPUT_TABLE, tableName);
    conf.setBoolean(RowInputFormat.CHECKPOINT_MODE, false);

    Job job = Job.getInstance(conf, "Busy Airport Count");

    job.setInputFormatClass(RowInputFormat.class);

    // configure mapper and reducer
    job.setMapperClass(SampleMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);

    // configure output
    TextOutputFormat.setOutputPath(job, outputPath);
    job.setOutputFormatClass(TextOutputFormat.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    return job.waitForCompletion(true) ? 0 : 1;
  }
開發者ID:gemxd，項目名稱:gemfirexd-oss，代碼行數:33，代碼來源:BusyAirports.java

示例12: run

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; //導入方法依賴的package包/類
public int run(String[] args) throws Exception {

    GfxdDataSerializable.initTypes();
    Configuration conf = getConf();

    Path outputPath = new Path(args[0]);
    String hdfsHomeDir = args[1];
    String tableName = args[2];

    outputPath.getFileSystem(conf).delete(outputPath, true);

    conf.set(RowInputFormat.HOME_DIR, hdfsHomeDir);
    conf.set(RowInputFormat.INPUT_TABLE, tableName);
    conf.setBoolean(RowInputFormat.CHECKPOINT_MODE, false);

    Job job = Job.getInstance(conf, "Busy Leg Count");

    job.setInputFormatClass(RowInputFormat.class);

    // configure mapper and reducer
    job.setMapperClass(SampleMapper.class);
    job.setCombinerClass(IntSumReducer.class);
    job.setReducerClass(IntSumReducer.class);

    // configure output
    TextOutputFormat.setOutputPath(job, outputPath);
    job.setOutputFormatClass(TextOutputFormat.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(IntWritable.class);

    return job.waitForCompletion(true) ? 0 : 1;
  }
開發者ID:gemxd，項目名稱:gemfirexd-oss，代碼行數:33，代碼來源:BusyLegs.java

示例13: start

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; //導入方法依賴的package包/類
public void start(Path outputDir, int numReducers, boolean concurrent) throws Exception {
  LOG.info("Running Verify with outputDir=" + outputDir +", numReducers=" + numReducers);
  
  DataStore<Long,CINode> store = DataStoreFactory.getDataStore(Long.class, CINode.class, new Configuration());

  job = new Job(getConf());
  
  if (!job.getConfiguration().get("io.serializations").contains("org.apache.hadoop.io.serializer.JavaSerialization")) {
    job.getConfiguration().set("io.serializations", job.getConfiguration().get("io.serializations") + ",org.apache.hadoop.io.serializer.JavaSerialization");
  }

  job.setJobName("Link Verifier");
  job.setNumReduceTasks(numReducers);
  job.setJarByClass(getClass());
  
  Query<Long,CINode> query = store.newQuery();
  if (!concurrent) {
    // no concurrency filtering, only need prev field
    query.setFields("prev");
  } else {
    readFlushed(job.getConfiguration());
  }

  GoraMapper.initMapperJob(job, query, store, LongWritable.class, VLongWritable.class, VerifyMapper.class, true);

  job.getConfiguration().setBoolean("mapred.map.tasks.speculative.execution", false);
  
  job.setReducerClass(VerifyReducer.class);
  job.setOutputFormatClass(TextOutputFormat.class);
  TextOutputFormat.setOutputPath(job, outputDir);

  store.close();
  
  job.submit();
}
開發者ID:jianglibo，項目名稱:gora-boot，代碼行數:36，代碼來源:Verify.java

示例14: run

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; //導入方法依賴的package包/類
@Override
public int run(String[] args) throws IOException, InterruptedException, ClassNotFoundException, AccumuloSecurityException {
  Job job = Job.getInstance(getConf());
  job.setJobName(this.getClass().getSimpleName() + "_" + System.currentTimeMillis());
  job.setJarByClass(this.getClass());
  Opts opts = new Opts();
  opts.parseArgs(getClass().getName(), args);

  job.setInputFormatClass(AccumuloInputFormat.class);
  opts.setAccumuloConfigs(job);

  HashSet<Pair<Text,Text>> columnsToFetch = new HashSet<>();
  for (String col : opts.columns.split(",")) {
    int idx = col.indexOf(":");
    Text cf = new Text(idx < 0 ? col : col.substring(0, idx));
    Text cq = idx < 0 ? null : new Text(col.substring(idx + 1));
    if (cf.getLength() > 0)
      columnsToFetch.add(new Pair<>(cf, cq));
  }
  if (!columnsToFetch.isEmpty())
    AccumuloInputFormat.fetchColumns(job, columnsToFetch);

  job.setMapperClass(TTFMapper.class);
  job.setMapOutputKeyClass(NullWritable.class);
  job.setMapOutputValueClass(Text.class);

  job.setNumReduceTasks(0);

  job.setOutputFormatClass(TextOutputFormat.class);
  TextOutputFormat.setOutputPath(job, new Path(opts.output));

  job.waitForCompletion(true);
  return job.isSuccessful() ? 0 : 1;
}
開發者ID:apache，項目名稱:accumulo-examples，代碼行數:35，代碼來源:TableToFile.java

示例15: run

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat; //導入方法依賴的package包/類
@Override
public int run(String[] args) throws Exception {
  Opts opts = new Opts();
  opts.parseArgs(getClass().getName(), args);

  Job job = Job.getInstance(getConf());
  job.setJobName(getClass().getSimpleName());
  job.setJarByClass(getClass());

  job.setInputFormatClass(AccumuloInputFormat.class);
  opts.setAccumuloConfigs(job);

  IteratorSetting regex = new IteratorSetting(50, "regex", RegExFilter.class);
  RegExFilter.setRegexs(regex, opts.rowRegex, opts.columnFamilyRegex, opts.columnQualifierRegex, opts.valueRegex, false);
  AccumuloInputFormat.addIterator(job, regex);

  job.setMapperClass(RegexMapper.class);
  job.setMapOutputKeyClass(Key.class);
  job.setMapOutputValueClass(Value.class);

  job.setNumReduceTasks(0);

  job.setOutputFormatClass(TextOutputFormat.class);
  TextOutputFormat.setOutputPath(job, new Path(opts.destination));

  System.out.println("setRowRegex: " + opts.rowRegex);
  System.out.println("setColumnFamilyRegex: " + opts.columnFamilyRegex);
  System.out.println("setColumnQualifierRegex: " + opts.columnQualifierRegex);
  System.out.println("setValueRegex: " + opts.valueRegex);

  job.waitForCompletion(true);
  return job.isSuccessful() ? 0 : 1;
}
開發者ID:apache，項目名稱:accumulo-examples，代碼行數:34，代碼來源:RegexExample.java


注：本文中的org.apache.hadoop.mapreduce.lib.output.TextOutputFormat.setOutputPath方法示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台，相關代碼片段篩選自各路編程大神貢獻的開源項目，源碼版權歸原作者所有，傳播和使用請參考對應項目的License；未經允許，請勿轉載。