本文整理汇总了Java中org.apache.hadoop.mapred.Task.CombineOutputCollector类的典型用法代码示例。如果您正苦于以下问题：Java CombineOutputCollector类的具体用法？Java CombineOutputCollector怎么用？Java CombineOutputCollector使用的例子？那么, 这里精选的类代码示例或许可以为您提供帮助。
CombineOutputCollector类属于org.apache.hadoop.mapred.Task包，在下文中一共展示了CombineOutputCollector类的7个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: testCustomCollect
import org.apache.hadoop.mapred.Task.CombineOutputCollector; //导入依赖的package包/类
@Test
public void testCustomCollect() throws Throwable {
//mock creation
TaskReporter mockTaskReporter = mock(TaskReporter.class);
@SuppressWarnings("unchecked")
Writer<String, Integer> mockWriter = mock(Writer.class);
Configuration conf = new Configuration();
conf.set(MRJobConfig.COMBINE_RECORDS_BEFORE_PROGRESS, "2");
coc = new CombineOutputCollector<String, Integer>(outCounter, mockTaskReporter, conf);
coc.setWriter(mockWriter);
verify(mockTaskReporter, never()).progress();
coc.collect("dummy", 1);
verify(mockTaskReporter, never()).progress();
coc.collect("dummy", 2);
verify(mockTaskReporter, times(1)).progress();
}
示例2: testDefaultCollect
import org.apache.hadoop.mapred.Task.CombineOutputCollector; //导入依赖的package包/类
@Test
public void testDefaultCollect() throws Throwable {
//mock creation
TaskReporter mockTaskReporter = mock(TaskReporter.class);
@SuppressWarnings("unchecked")
Writer<String, Integer> mockWriter = mock(Writer.class);
Configuration conf = new Configuration();
coc = new CombineOutputCollector<String, Integer>(outCounter, mockTaskReporter, conf);
coc.setWriter(mockWriter);
verify(mockTaskReporter, never()).progress();
for(int i = 0; i < Task.DEFAULT_COMBINE_RECORDS_BEFORE_PROGRESS; i++) {
coc.collect("dummy", i);
}
verify(mockTaskReporter, times(1)).progress();
for(int i = 0; i < Task.DEFAULT_COMBINE_RECORDS_BEFORE_PROGRESS; i++) {
coc.collect("dummy", i);
}
verify(mockTaskReporter, times(2)).progress();
}
示例3: testCustomCollect
import org.apache.hadoop.mapred.Task.CombineOutputCollector; //导入依赖的package包/类
@Test
public void testCustomCollect() throws Throwable {
//mock creation
TaskReporter mockTaskReporter = mock(TaskReporter.class);
Counters.Counter outCounter = new Counters.Counter();
Writer<String, Integer> mockWriter = mock(Writer.class);
Configuration conf = new Configuration();
conf.set("mapred.combine.recordsBeforeProgress", "2");
coc = new CombineOutputCollector<String, Integer>(outCounter, mockTaskReporter, conf);
coc.setWriter(mockWriter);
verify(mockTaskReporter, never()).progress();
coc.collect("dummy", 1);
verify(mockTaskReporter, never()).progress();
coc.collect("dummy", 2);
verify(mockTaskReporter, times(1)).progress();
}
示例4: testDefaultCollect
import org.apache.hadoop.mapred.Task.CombineOutputCollector; //导入依赖的package包/类
@Test
public void testDefaultCollect() throws Throwable {
//mock creation
TaskReporter mockTaskReporter = mock(TaskReporter.class);
Counters.Counter outCounter = new Counters.Counter();
Writer<String, Integer> mockWriter = mock(Writer.class);
Configuration conf = new Configuration();
coc = new CombineOutputCollector<String, Integer>(outCounter, mockTaskReporter, conf);
coc.setWriter(mockWriter);
verify(mockTaskReporter, never()).progress();
for(int i = 0; i < Task.DEFAULT_MR_COMBINE_RECORDS_BEFORE_PROGRESS; i++) {
coc.collect("dummy", i);
}
verify(mockTaskReporter, times(1)).progress();
for(int i = 0; i < Task.DEFAULT_MR_COMBINE_RECORDS_BEFORE_PROGRESS; i++) {
coc.collect("dummy", i);
}
verify(mockTaskReporter, times(2)).progress();
}
示例5: MapSpiller
import org.apache.hadoop.mapred.Task.CombineOutputCollector; //导入依赖的package包/类
MapSpiller(JobConf job,TaskAttemptID tid, TaskReporter rep) throws ClassNotFoundException {
reporter = rep;
conf = job;
this.taskId = tid;
mapOutputFile.setConf(conf);
mapOutputByteCounter = reporter.getCounter(MAP_OUTPUT_BYTES);
Counters.Counter combineInputCounter =
reporter.getCounter(COMBINE_INPUT_RECORDS);
combineOutputCounter = reporter.getCounter(COMBINE_OUTPUT_RECORDS);
fileOutputByteCounter = reporter.getCounter(MAP_OUTPUT_MATERIALIZED_BYTES);
// combiner
combinerRunner = CombinerRunner.create(conf, taskId,
combineInputCounter,
reporter, null);
if (combinerRunner != null) {
combineCollector= new CombineOutputCollector(combineOutputCounter, reporter, conf);
} else {
combineCollector = null;
}
indexCacheList = new ArrayList<SpillRecord>();
spilledRecordsCounter = reporter.getCounter(Counter.SPILLED_RECORDS);
}
示例6: Context
import org.apache.hadoop.mapred.Task.CombineOutputCollector; //导入依赖的package包/类
public Context(org.apache.hadoop.mapreduce.TaskAttemptID reduceId,
JobConf jobConf, FileSystem localFS,
TaskUmbilicalProtocol umbilical,
LocalDirAllocator localDirAllocator,
Reporter reporter, CompressionCodec codec,
Class<? extends Reducer> combinerClass,
CombineOutputCollector<K,V> combineCollector,
Counters.Counter spilledRecordsCounter,
Counters.Counter reduceCombineInputCounter,
Counters.Counter shuffledMapsCounter,
Counters.Counter reduceShuffleBytes,
Counters.Counter failedShuffleCounter,
Counters.Counter mergedMapOutputsCounter,
TaskStatus status, Progress copyPhase, Progress mergePhase,
Task reduceTask, MapOutputFile mapOutputFile,
Map<TaskAttemptID, MapOutputFile> localMapFiles) {
this.reduceId = reduceId;
this.jobConf = jobConf;
this.localFS = localFS;
this. umbilical = umbilical;
this.localDirAllocator = localDirAllocator;
this.reporter = reporter;
this.codec = codec;
this.combinerClass = combinerClass;
this.combineCollector = combineCollector;
this.spilledRecordsCounter = spilledRecordsCounter;
this.reduceCombineInputCounter = reduceCombineInputCounter;
this.shuffledMapsCounter = shuffledMapsCounter;
this.reduceShuffleBytes = reduceShuffleBytes;
this.failedShuffleCounter = failedShuffleCounter;
this.mergedMapOutputsCounter = mergedMapOutputsCounter;
this.status = status;
this.copyPhase = copyPhase;
this.mergePhase = mergePhase;
this.reduceTask = reduceTask;
this.mapOutputFile = mapOutputFile;
this.localMapFiles = localMapFiles;
}
示例7: Context
import org.apache.hadoop.mapred.Task.CombineOutputCollector; //导入依赖的package包/类
public Context(org.apache.hadoop.mapreduce.TaskAttemptID reduceId,
JobConf jobConf, FileSystem localFS,
TaskUmbilicalProtocol umbilical,
LocalDirAllocator localDirAllocator,
Reporter reporter, CompressionCodec codec,
Class<? extends Reducer> combinerClass,
CombineOutputCollector<K,V> combineCollector,
Counters.Counter spilledRecordsCounter,
Counters.Counter reduceCombineInputCounter,
Counters.Counter shuffledMapsCounter,
Counters.Counter reduceShuffleBytes,
Counters.Counter failedShuffleCounter,
Counters.Counter mergedMapOutputsCounter,
TaskStatus status, Progress copyPhase, Progress mergePhase,
Task reduceTask, MapOutputFile mapOutputFile) {
this.reduceId = reduceId;
this.jobConf = jobConf;
this.localFS = localFS;
this. umbilical = umbilical;
this.localDirAllocator = localDirAllocator;
this.reporter = reporter;
this.codec = codec;
this.combinerClass = combinerClass;
this.combineCollector = combineCollector;
this.spilledRecordsCounter = spilledRecordsCounter;
this.reduceCombineInputCounter = reduceCombineInputCounter;
this.shuffledMapsCounter = shuffledMapsCounter;
this.reduceShuffleBytes = reduceShuffleBytes;
this.failedShuffleCounter = failedShuffleCounter;
this.mergedMapOutputsCounter = mergedMapOutputsCounter;
this.status = status;
this.copyPhase = copyPhase;
this.mergePhase = mergePhase;
this.reduceTask = reduceTask;
this.mapOutputFile = mapOutputFile;
}