當前位置: 首頁>>代碼示例>>Java>>正文

Java MultipleOutputs.setCountersEnabled方法代碼示例

本文整理匯總了Java中org.apache.hadoop.mapreduce.lib.output.MultipleOutputs.setCountersEnabled方法的典型用法代碼示例。如果您正苦於以下問題：Java MultipleOutputs.setCountersEnabled方法的具體用法？Java MultipleOutputs.setCountersEnabled怎麽用？Java MultipleOutputs.setCountersEnabled使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類org.apache.hadoop.mapreduce.lib.output.MultipleOutputs的用法示例。


在下文中一共展示了MultipleOutputs.setCountersEnabled方法的7個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Java代碼示例。

示例1: configureTextOutput

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入方法依賴的package包/類
/**
 * Set up a MapReduce job to output human-readable text.
 */
protected void configureTextOutput(String destination) {
    Path outPath;
    outPath = MRReasoningUtils.getOutputPath(job.getConfiguration(), destination);
    TextOutputFormat.setOutputPath(job, outPath);
    LazyOutputFormat.setOutputFormatClass(job, TextOutputFormat.class);
    MultipleOutputs.addNamedOutput(job, MRReasoningUtils.INTERMEDIATE_OUT,
        TextOutputFormat.class, NullWritable.class, Text.class);
    MultipleOutputs.addNamedOutput(job, MRReasoningUtils.TERMINAL_OUT,
        TextOutputFormat.class, NullWritable.class, Text.class);
    MultipleOutputs.addNamedOutput(job, MRReasoningUtils.SCHEMA_OUT,
        TextOutputFormat.class, NullWritable.class, Text.class);
    MultipleOutputs.addNamedOutput(job, MRReasoningUtils.INCONSISTENT_OUT,
        TextOutputFormat.class, NullWritable.class, Text.class);
    MultipleOutputs.addNamedOutput(job, MRReasoningUtils.DEBUG_OUT,
        TextOutputFormat.class, Text.class, Text.class);
    MultipleOutputs.setCountersEnabled(job, true);
}
開發者ID:apache，項目名稱:incubator-rya，代碼行數:21，代碼來源:AbstractReasoningTool.java

示例2: configureSchemaOutput

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入方法依賴的package包/類
/**
 * Set up the MapReduce job to output a schema (TBox).
 */
protected void configureSchemaOutput() {
    Path outPath = MRReasoningUtils.getSchemaPath(job.getConfiguration());
    SequenceFileOutputFormat.setOutputPath(job, outPath);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);
    job.setOutputKeyClass(NullWritable.class);
    job.setOutputValueClass(SchemaWritable.class);
    LazyOutputFormat.setOutputFormatClass(job, SequenceFileOutputFormat.class);
    MultipleOutputs.addNamedOutput(job, "schemaobj",
        SequenceFileOutputFormat.class, NullWritable.class, SchemaWritable.class);
    MultipleOutputs.addNamedOutput(job, MRReasoningUtils.DEBUG_OUT,
        TextOutputFormat.class, Text.class, Text.class);
    MultipleOutputs.setCountersEnabled(job, true);
}
開發者ID:apache，項目名稱:incubator-rya，代碼行數:17，代碼來源:AbstractReasoningTool.java

示例3: configureDerivationOutput

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入方法依賴的package包/類
/**
 * Set up a MapReduce job to output newly derived triples.
 * @param   intermediate    True if this is intermediate data. Outputs
 *                          to [base]-[iteration]-[temp].
 */
protected void configureDerivationOutput(boolean intermediate) {
    Path outPath;
    Configuration conf = job.getConfiguration();
    int iteration = MRReasoningUtils.getCurrentIteration(conf);
    if (intermediate) {
        outPath = MRReasoningUtils.getOutputPath(conf,
            MRReasoningUtils.OUTPUT_BASE + iteration
            + MRReasoningUtils.TEMP_SUFFIX);
    }
    else {
        outPath = MRReasoningUtils.getOutputPath(conf,
            MRReasoningUtils.OUTPUT_BASE + iteration);
    }
    SequenceFileOutputFormat.setOutputPath(job, outPath);
    LazyOutputFormat.setOutputFormatClass(job, SequenceFileOutputFormat.class);
    MultipleOutputs.addNamedOutput(job, MRReasoningUtils.INTERMEDIATE_OUT,
        SequenceFileOutputFormat.class, Fact.class, NullWritable.class);
    MultipleOutputs.addNamedOutput(job, MRReasoningUtils.TERMINAL_OUT,
        SequenceFileOutputFormat.class, Fact.class, NullWritable.class);
    MultipleOutputs.addNamedOutput(job, MRReasoningUtils.SCHEMA_OUT,
        SequenceFileOutputFormat.class, Fact.class, NullWritable.class);
    MultipleOutputs.addNamedOutput(job, MRReasoningUtils.INCONSISTENT_OUT,
        SequenceFileOutputFormat.class, Derivation.class, NullWritable.class);
    MultipleOutputs.setCountersEnabled(job, true);
    // Set up an output for diagnostic info, if needed
    MultipleOutputs.addNamedOutput(job, MRReasoningUtils.DEBUG_OUT,
        TextOutputFormat.class, Text.class, Text.class);
}
開發者ID:apache，項目名稱:incubator-rya，代碼行數:34，代碼來源:AbstractReasoningTool.java

示例4: run

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入方法依賴的package包/類
@Override
public int run(String[] args) throws Exception {
	Configuration conf = new Configuration();
	GenericOptionsParser parser = new GenericOptionsParser(conf, args);
	String[] otherArgs = parser.getRemainingArgs();
	if (otherArgs.length != 2) {
		System.err.println("Usage: BinningTags <in> <out>");
		ToolRunner.printGenericCommandUsage(System.err);
		System.exit(2);
	}
	Job job = new Job(conf, "Binning Tags");
	job.setJarByClass(BinningTags.class);
	// Configure the MultipleOutputs by adding an output called "bins"
	// With the proper output format and mapper key/value pairs
	MultipleOutputs.addNamedOutput(job, "bins", TextOutputFormat.class,
			Text.class, NullWritable.class);
	// Enable the counters for the job
	// If there are a significant number of different named outputs, this
	// should be disabled
	MultipleOutputs.setCountersEnabled(job, true);
	// Map-only job
	job.setNumReduceTasks(0);
	job.setMapperClass(BinningMapper.class);
	job.setMapOutputKeyClass(Text.class);
	job.setMapOutputValueClass(NullWritable.class);
	job.setOutputKeyClass(Text.class);
	job.setOutputValueClass(NullWritable.class);
	FileInputFormat.addInputPath(job, new Path(otherArgs[0]));
	FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));
	boolean success = job.waitForCompletion(true);
	return success ? 0 : 1;
}
開發者ID:geftimov，項目名稱:hadoop-map-reduce-patterns，代碼行數:33，代碼來源:BinningTags.java

示例5: _testMultipleOutputs

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入方法依賴的package包/類
protected void _testMultipleOutputs(boolean withCounters) throws Exception {
  Path inDir = getDir(IN_DIR);
  Path outDir = getDir(OUT_DIR);

  JobConf conf = createJobConf();
  FileSystem fs = FileSystem.get(conf);

  DataOutputStream file = fs.create(new Path(inDir, "part-0"));
  file.writeBytes("a\nb\n\nc\nd\ne");
  file.close();

  file = fs.create(new Path(inDir, "part-1"));
  file.writeBytes("a\nb\n\nc\nd\ne");
  file.close();

  Job job = new Job(conf);
  job.setJobName("mo");
  job.setInputFormatClass(TextInputFormat.class);

  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(Text.class);

  job.setMapOutputKeyClass(LongWritable.class);
  job.setMapOutputValueClass(Text.class);

  job.setOutputFormatClass(TextOutputFormat.class);
  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(Text.class);

  MultipleOutputs.addNamedOutput(job, "text", TextOutputFormat.class,
    LongWritable.class, Text.class);

  MultipleOutputs.setCountersEnabled(job, withCounters);

  job.setMapperClass(MOMap.class);
  job.setReducerClass(MOReduce.class);

  FileInputFormat.setInputPaths(job, inDir);
  FileOutputFormat.setOutputPath(job, outDir);

  job.waitForCompletion(false);

  // assert number of named output part files
  int namedOutputCount = 0;
  FileStatus[] statuses = fs.listStatus(outDir);
  
  for (FileStatus status : statuses) {
    if (status.getPath().getName().equals("text-m-00000") ||
      status.getPath().getName().equals("text-m-00001") ||
      status.getPath().getName().equals("text-r-00000")) {
      namedOutputCount++;
    }
  }
  assertEquals(3, namedOutputCount);

  // assert TextOutputFormat files correctness
  JobContext jobContext = new JobContextImpl(job.getConfiguration(), new JobID());
  BufferedReader reader = new BufferedReader(
    new InputStreamReader(fs.open(
      new Path(FileOutputFormat.getOutputPath(jobContext), "text-r-00000"))));
  int count = 0;
  String line = reader.readLine();
  while (line != null) {
    assertTrue(line.endsWith("text"));
    line = reader.readLine();
    count++;
  }
  reader.close();
  assertFalse(count == 0);

  CounterGroup counters =
    job.getCounters().getGroup(MultipleOutputs.class.getName());
  if (!withCounters) {
    assertEquals(0, counters.size());
  }
  else {
    assertEquals(1, counters.size());
    assertEquals(4, counters.findCounter("text").getValue());
  }

}
開發者ID:Nextzero，項目名稱:hadoop-2.6.0-cdh5.4.3，代碼行數:82，代碼來源:TestMultipleOutputs.java

示例6: process

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入方法依賴的package包/類
@Override
public void process(Annotation annotation, Job job, Object target)
		throws ToolException {
	NamedOutput namedOut = (NamedOutput)annotation;
	KeyValue kv = namedOut.type();

	// If this is a MultipleOutputs member we're annotating, see if we can't
	// get the key/value from the parameters if there are any.
	Pair<Type, Type> kvTypePair = getGenericTypeParams(target);

	Class<?> keyClass = kv.key();
	if (keyClass == void.class) {
		if (kvTypePair != null) {
			keyClass = (Class<?>)kvTypePair.getKey();
		}
		else {
			// fall back on job output key class
			keyClass = job.getOutputKeyClass();
		}
	}

	Class<?> valueClass = kv.value();
	if (valueClass == void.class) {
		if (kvTypePair != null) {
			valueClass = (Class<?>)kvTypePair.getValue();
		}
		else {
			valueClass = job.getOutputValueClass();
		}
	}

	String[] names = getNames(namedOut);
	for (String name : names) {
		name = (String)evaluateExpression(name);
		if (!configured.contains(name)) {
			MultipleOutputs.addNamedOutput(job, name, namedOut.format(), keyClass, valueClass);
			MultipleOutputs.setCountersEnabled(job, namedOut.countersEnabled());
			configured.add(name);
		}
	}
}
開發者ID:conversant，項目名稱:mara，代碼行數:42，代碼來源:NamedOutputAnnotationHandler.java

示例7: _testMultipleOutputs

import org.apache.hadoop.mapreduce.lib.output.MultipleOutputs; //導入方法依賴的package包/類
protected void _testMultipleOutputs(boolean withCounters) throws Exception {
  Path inDir = getDir(IN_DIR);
  Path outDir = getDir(OUT_DIR);

  JobConf conf = createJobConf();
  FileSystem fs = FileSystem.get(conf);

  DataOutputStream file = fs.create(new Path(inDir, "part-0"));
  file.writeBytes("a\nb\n\nc\nd\ne");
  file.close();

  file = fs.create(new Path(inDir, "part-1"));
  file.writeBytes("a\nb\n\nc\nd\ne");
  file.close();

  Job job = new Job(conf);
  job.setJobName("mo");
  job.setInputFormatClass(TextInputFormat.class);

  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(Text.class);

  job.setMapOutputKeyClass(LongWritable.class);
  job.setMapOutputValueClass(Text.class);

  job.setOutputFormatClass(TextOutputFormat.class);
  job.setOutputKeyClass(LongWritable.class);
  job.setOutputValueClass(Text.class);

  MultipleOutputs.addNamedOutput(job, "text", TextOutputFormat.class,
    LongWritable.class, Text.class);

  MultipleOutputs.setCountersEnabled(job, withCounters);

  job.setMapperClass(MOMap.class);
  job.setReducerClass(MOReduce.class);

  FileInputFormat.setInputPaths(job, inDir);
  FileOutputFormat.setOutputPath(job, outDir);

  job.waitForCompletion(false);

  // assert number of named output part files
  int namedOutputCount = 0;
  FileStatus[] statuses = fs.listStatus(outDir);
  
  for (FileStatus status : statuses) {
    if (status.getPath().getName().equals("text-m-00000") ||
      status.getPath().getName().equals("text-m-00001") ||
      status.getPath().getName().equals("text-r-00000")) {
      namedOutputCount++;
    }
  }
  assertEquals(3, namedOutputCount);

  // assert TextOutputFormat files correctness
  JobContext jobContext = new JobContext(job.getConfiguration(), new JobID());
  BufferedReader reader = new BufferedReader(
    new InputStreamReader(fs.open(
      new Path(FileOutputFormat.getOutputPath(jobContext), "text-r-00000"))));
  int count = 0;
  String line = reader.readLine();
  while (line != null) {
    assertTrue(line.endsWith("text"));
    line = reader.readLine();
    count++;
  }
  reader.close();
  assertFalse(count == 0);

  CounterGroup counters =
    job.getCounters().getGroup(MultipleOutputs.class.getName());
  if (!withCounters) {
    assertEquals(0, counters.size());
  }
  else {
    assertEquals(1, counters.size());
    assertEquals(4, counters.findCounter("text").getValue());
  }

}
開發者ID:Seagate，項目名稱:hadoop-on-lustre，代碼行數:82，代碼來源:TestMultipleOutputs.java


注：本文中的org.apache.hadoop.mapreduce.lib.output.MultipleOutputs.setCountersEnabled方法示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台，相關代碼片段篩選自各路編程大神貢獻的開源項目，源碼版權歸原作者所有，傳播和使用請參考對應項目的License；未經允許，請勿轉載。