本文整理汇总了Java中org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter类的典型用法代码示例。如果您正苦于以下问题：Java FileOutputFormatCounter类的具体用法？Java FileOutputFormatCounter怎么用？Java FileOutputFormatCounter使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
FileOutputFormatCounter类属于org.apache.hadoop.mapreduce.lib.output包，在下文中一共展示了FileOutputFormatCounter类的4个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: validateFileCounters
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter; //导入依赖的package包/类
private void validateFileCounters(Counters counter, long fileBytesRead,
long fileBytesWritten, long mapOutputBytes,
long mapOutputMaterializedBytes) {
assertTrue(counter.findCounter(FileInputFormatCounter.BYTES_READ)
.getValue() != 0);
assertEquals(fileBytesRead,
counter.findCounter(FileInputFormatCounter.BYTES_READ).getValue());
assertTrue(counter.findCounter(FileOutputFormatCounter.BYTES_WRITTEN)
.getValue() != 0);
if (mapOutputBytes >= 0) {
assertTrue(counter.findCounter(TaskCounter.MAP_OUTPUT_BYTES).getValue() != 0);
}
if (mapOutputMaterializedBytes >= 0) {
assertTrue(counter.findCounter(TaskCounter.MAP_OUTPUT_MATERIALIZED_BYTES)
.getValue() != 0);
}
}
示例2: NewDirectOutputCollector
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter; //导入依赖的package包/类
@SuppressWarnings("unchecked")
NewDirectOutputCollector(MRJobConfig jobContext,
JobConf job, TaskUmbilicalProtocol umbilical, TaskReporter reporter)
throws IOException, ClassNotFoundException, InterruptedException {
this.reporter = reporter;
mapOutputRecordCounter = reporter
.getCounter(TaskCounter.MAP_OUTPUT_RECORDS);
fileOutputByteCounter = reporter
.getCounter(FileOutputFormatCounter.BYTES_WRITTEN);
List<Statistics> matchedStats = null;
if (outputFormat instanceof org.apache.hadoop.mapreduce.lib.output.FileOutputFormat) {
matchedStats = getFsStatistics(org.apache.hadoop.mapreduce.lib.output.FileOutputFormat
.getOutputPath(taskContext), taskContext.getConfiguration());
}
fsStats = matchedStats;
long bytesOutPrev = getOutputBytes(fsStats);
out = outputFormat.getRecordWriter(taskContext);
long bytesOutCurr = getOutputBytes(fsStats);
fileOutputByteCounter.increment(bytesOutCurr - bytesOutPrev);
}
示例3: init
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter; //导入依赖的package包/类
@SuppressWarnings("unchecked")
public void init(MapOutputCollector.Context context
) throws IOException, ClassNotFoundException {
this.reporter = context.getReporter();
JobConf job = context.getJobConf();
String finalName = getOutputName(getPartition());
FileSystem fs = FileSystem.get(job);
OutputFormat<K, V> outputFormat = job.getOutputFormat();
mapOutputRecordCounter = reporter.getCounter(TaskCounter.MAP_OUTPUT_RECORDS);
fileOutputByteCounter = reporter
.getCounter(FileOutputFormatCounter.BYTES_WRITTEN);
List<Statistics> matchedStats = null;
if (outputFormat instanceof FileOutputFormat) {
matchedStats = getFsStatistics(FileOutputFormat.getOutputPath(job), job);
}
fsStats = matchedStats;
long bytesOutPrev = getOutputBytes(fsStats);
out = job.getOutputFormat().getRecordWriter(fs, job, finalName, reporter);
long bytesOutCurr = getOutputBytes(fsStats);
fileOutputByteCounter.increment(bytesOutCurr - bytesOutPrev);
}
示例4: initDepricatedMap
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter; //导入依赖的package包/类
@SuppressWarnings({ "deprecation" })
private static void initDepricatedMap() {
depricatedCounterMap.put(FileInputFormat.Counter.class.getName(),
FileInputFormatCounter.class.getName());
depricatedCounterMap.put(FileOutputFormat.Counter.class.getName(),
FileOutputFormatCounter.class.getName());
depricatedCounterMap.put(
org.apache.hadoop.mapreduce.lib.input.FileInputFormat.Counter.class
.getName(), FileInputFormatCounter.class.getName());
depricatedCounterMap.put(
org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.Counter.class
.getName(), FileOutputFormatCounter.class.getName());
}