Java TezCounter类代码示例

import org.apache.tez.common.counters.TezCounter; //导入依赖的package包/类
public ShuffledUnorderedKVReader(ShuffleManager shuffleManager, Configuration conf,
    CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength, int ifileBufferSize,
    TezCounter inputRecordCounter)
    throws IOException {
  this.shuffleManager = shuffleManager;

  this.codec = codec;
  this.ifileReadAhead = ifileReadAhead;
  this.ifileReadAheadLength = ifileReadAheadLength;
  this.ifileBufferSize = ifileBufferSize;
  this.inputRecordCounter = inputRecordCounter;

  this.keyClass = ConfigUtils.getIntermediateInputKeyClass(conf);
  this.valClass = ConfigUtils.getIntermediateInputValueClass(conf);

  this.keyIn = new DataInputBuffer();
  this.valIn = new DataInputBuffer();

  SerializationFactory serializationFactory = new SerializationFactory(conf);

  this.keyDeserializer = serializationFactory.getDeserializer(keyClass);
  this.keyDeserializer.open(keyIn);
  this.valDeserializer = serializationFactory.getDeserializer(valClass);
  this.valDeserializer.open(valIn);
}
 

import org.apache.tez.common.counters.TezCounter; //导入依赖的package包/类
public static
TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
                          Class keyClass, Class valueClass, 
                          CompressionCodec codec, boolean ifileReadAhead,
                          int ifileReadAheadLength, int ifileBufferSize,
                          Path[] inputs, boolean deleteInputs, 
                          int mergeFactor, Path tmpDir,
                          RawComparator comparator, Progressable reporter,
                          TezCounter readsCounter,
                          TezCounter writesCounter,
                          TezCounter bytesReadCounter,
                          Progress mergePhase)
throws IOException {
  return 
    new MergeQueue(conf, fs, inputs, deleteInputs, codec, ifileReadAhead,
                         ifileReadAheadLength, ifileBufferSize, false, comparator, 
                         reporter, null).merge(keyClass, valueClass,
                                         mergeFactor, tmpDir,
                                         readsCounter, writesCounter,
                                         bytesReadCounter,
                                         mergePhase);
}
 

import org.apache.tez.common.counters.TezCounter; //导入依赖的package包/类
public Segment(Configuration conf, FileSystem fs, Path file,
    long segmentOffset, long segmentLength, CompressionCodec codec,
    boolean ifileReadAhead, int ifileReadAheadLength, int bufferSize,
    boolean preserve, TezCounter mergedMapOutputsCounter)
throws IOException {
  this.conf = conf;
  this.fs = fs;
  this.file = file;
  this.codec = codec;
  this.preserve = preserve;
  this.ifileReadAhead = ifileReadAhead;
  this.ifileReadAheadLength =ifileReadAheadLength;
  this.bufferSize = bufferSize;

  this.segmentOffset = segmentOffset;
  this.segmentLength = segmentLength;
  
  this.mapOutputsCounter = mergedMapOutputsCounter;
}
 

import org.apache.tez.common.counters.TezCounter; //导入依赖的package包/类
public ValuesIterator (TezRawKeyValueIterator in, 
                       RawComparator<KEY> comparator, 
                       Class<KEY> keyClass,
                       Class<VALUE> valClass, Configuration conf,
                       TezCounter inputKeyCounter,
                       TezCounter inputValueCounter)
  throws IOException {
  this.in = in;
  this.comparator = comparator;
  this.inputKeyCounter = inputKeyCounter;
  this.inputValueCounter = inputValueCounter;
  SerializationFactory serializationFactory = new SerializationFactory(conf);
  this.keyDeserializer = serializationFactory.getDeserializer(keyClass);
  this.keyDeserializer.open(keyIn);
  this.valDeserializer = serializationFactory.getDeserializer(valClass);
  this.valDeserializer.open(this.valueIn);
}
 

import org.apache.tez.common.counters.TezCounter; //导入依赖的package包/类
public final TezCounter getLocalityInfo() {
  Map<String, TezCounter> dataLocalTask = getCounter(DAGCounter.class.getName(),
      DAGCounter.DATA_LOCAL_TASKS.toString());
  Map<String, TezCounter> rackLocalTask = getCounter(DAGCounter.class.getName(),
      DAGCounter.RACK_LOCAL_TASKS.toString());
  Map<String, TezCounter> otherLocalTask = getCounter(DAGCounter.class.getName(),
      DAGCounter.OTHER_LOCAL_TASKS.toString());

  if (!dataLocalTask.isEmpty()) {
    return dataLocalTask.get(DAGCounter.class.getName());
  }

  if (!rackLocalTask.isEmpty()) {
    return rackLocalTask.get(DAGCounter.class.getName());
  }

  if (!otherLocalTask.isEmpty()) {
    return otherLocalTask.get(DAGCounter.class.getName());
  }
  return null;
}
 

import org.apache.tez.common.counters.TezCounter; //导入依赖的package包/类
public static Counters fromTez(TezCounters tezCounters) {
  if (tezCounters == null) {
    return null;
  }
  Counters counters = new Counters();
  for (CounterGroup xGrp : tezCounters) {
    counters.addGroup(xGrp.getName(), xGrp.getDisplayName());
    for (TezCounter xCounter : xGrp) {
      Counter counter =
          counters.findCounter(xGrp.getName(), xCounter.getName());
      counter.setValue(xCounter.getValue());

    }
  }
  return counters;
}
 

import org.apache.tez.common.counters.TezCounter; //导入依赖的package包/类
public MRReaderMapReduce(JobConf jobConf, InputSplit inputSplit, TezCounters tezCounters,
    TezCounter inputRecordCounter, long clusterId, int vertexIndex, int appId, int taskIndex,
    int taskAttemptNumber) throws IOException {
  this.inputRecordCounter = inputRecordCounter;
  this.taskAttemptContext = new TaskAttemptContextImpl(jobConf, tezCounters, clusterId,
      vertexIndex, appId, taskIndex, taskAttemptNumber, true, null);

  Class<? extends org.apache.hadoop.mapreduce.InputFormat<?, ?>> inputFormatClazz;
  try {
    inputFormatClazz = taskAttemptContext.getInputFormatClass();
  } catch (ClassNotFoundException e) {
    throw new IOException("Unable to instantiate InputFormat class", e);
  }
  inputFormat = ReflectionUtils.newInstance(inputFormatClazz, jobConf);

  if (inputSplit != null) {
    this.inputSplit = inputSplit;
    setupNewRecordReader();
  }
}
 

import org.apache.tez.common.counters.TezCounter; //导入依赖的package包/类
public static Map<String,Object> convertCountersToATSMap(TezCounters counters) {
  Map<String,Object> object = new LinkedHashMap<String, Object>();
  if (counters == null) {
      return object;
    }
  ArrayList<Object> counterGroupsList = new ArrayList<Object>();
  for (CounterGroup group : counters) {
      Map<String,Object> counterGroupMap = new LinkedHashMap<String, Object>();
      counterGroupMap.put(ATSConstants.COUNTER_GROUP_NAME, group.getName());
      counterGroupMap.put(ATSConstants.COUNTER_GROUP_DISPLAY_NAME,
              group.getDisplayName());
      ArrayList<Object> counterList = new ArrayList<Object>();
      for (TezCounter counter : group) {
          Map<String,Object> counterMap = new LinkedHashMap<String, Object>();
          counterMap.put(ATSConstants.COUNTER_NAME, counter.getName());
          counterMap.put(ATSConstants.COUNTER_DISPLAY_NAME,
                  counter.getDisplayName());
          counterMap.put(ATSConstants.COUNTER_VALUE, counter.getValue());
          counterList.add(counterMap);
        }
      putInto(counterGroupMap, ATSConstants.COUNTERS, counterList);
      counterGroupsList.add(counterGroupMap);
    }
  putInto(object, ATSConstants.COUNTER_GROUPS, counterGroupsList);
  return object;
}
 

import org.apache.tez.common.counters.TezCounter; //导入依赖的package包/类
public UnorderedKVReader(ShuffleManager shuffleManager, Configuration conf,
    CompressionCodec codec, boolean ifileReadAhead, int ifileReadAheadLength, int ifileBufferSize,
    TezCounter inputRecordCounter, InputContext context)
    throws IOException {
  this.shuffleManager = shuffleManager;
  this.context = context;
  this.codec = codec;
  this.ifileReadAhead = ifileReadAhead;
  this.ifileReadAheadLength = ifileReadAheadLength;
  this.ifileBufferSize = ifileBufferSize;
  this.inputRecordCounter = inputRecordCounter;

  this.keyClass = ConfigUtils.getIntermediateInputKeyClass(conf);
  this.valClass = ConfigUtils.getIntermediateInputValueClass(conf);

  this.keyIn = new DataInputBuffer();
  this.valIn = new DataInputBuffer();

  SerializationFactory serializationFactory = new SerializationFactory(conf);

  this.keyDeserializer = serializationFactory.getDeserializer(keyClass);
  this.keyDeserializer.open(keyIn);
  this.valDeserializer = serializationFactory.getDeserializer(valClass);
  this.valDeserializer.open(valIn);
}
 

import org.apache.tez.common.counters.TezCounter; //导入依赖的package包/类
public static
TezRawKeyValueIterator merge(Configuration conf, FileSystem fs,
                          Class keyClass, Class valueClass, 
                          CompressionCodec codec, boolean ifileReadAhead,
                          int ifileReadAheadLength, int ifileBufferSize,
                          Path[] inputs, boolean deleteInputs, 
                          int mergeFactor, Path tmpDir,
                          RawComparator comparator, Progressable reporter,
                          TezCounter readsCounter,
                          TezCounter writesCounter,
                          TezCounter bytesReadCounter,
                          Progress mergePhase)
    throws IOException, InterruptedException {
  return 
    new MergeQueue(conf, fs, inputs, deleteInputs, codec, ifileReadAhead,
                         ifileReadAheadLength, ifileBufferSize, false, comparator, 
                         reporter, null).merge(keyClass, valueClass,
                                         mergeFactor, tmpDir,
                                         readsCounter, writesCounter,
                                         bytesReadCounter,
                                         mergePhase);
}
 

import org.apache.tez.common.counters.TezCounter; //导入依赖的package包/类
public DiskSegment(FileSystem fs, Path file,
    long segmentOffset, long segmentLength, CompressionCodec codec,
    boolean ifileReadAhead, int ifileReadAheadLength, int bufferSize,
    boolean preserve, TezCounter mergedMapOutputsCounter)
throws IOException {
  super(null, mergedMapOutputsCounter);
  this.fs = fs;
  this.file = file;
  this.codec = codec;
  this.preserve = preserve;
  this.ifileReadAhead = ifileReadAhead;
  this.ifileReadAheadLength =ifileReadAheadLength;
  this.bufferSize = bufferSize;

  this.segmentOffset = segmentOffset;
  this.segmentLength = segmentLength;
}
 

import org.apache.tez.common.counters.TezCounter; //导入依赖的package包/类
@Test(timeout = 5000)
public void testInterruptOnNext() throws IOException, InterruptedException {
  ShuffleManager shuffleManager = mock(ShuffleManager.class);

  // Simulate an interrupt while waiting for the next fetched input.
  doThrow(new InterruptedException()).when(shuffleManager).getNextInput();
  TezCounters counters = new TezCounters();
  TezCounter inputRecords = counters.findCounter(TaskCounter.INPUT_RECORDS_PROCESSED);
  UnorderedKVReader<Text, Text> reader =
      new UnorderedKVReader<Text, Text>(shuffleManager, defaultConf, null, false, -1, -1,
          inputRecords, mock(InputContext.class));

  try {
    reader.next();
    fail("No data available to reader. Should not be able to access any record");
  } catch (IOInterruptedException e) {
    // Expected exception. Any other should fail the test.
  }
}
 

import org.apache.tez.common.counters.TezCounter; //导入依赖的package包/类
@Test
  public void testEmptyDataWithPipelinedShuffle() throws IOException {
    this.numOutputs = 1;
    this.initialAvailableMem = 1 *1024 * 1024;
    conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, false);
    conf.setInt(TezRuntimeConfiguration
        .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, 1);
    PipelinedSorter sorter = new PipelinedSorter(this.outputContext, conf, numOutputs,
        initialAvailableMem);

    writeData(sorter, 0, 1<<20);

    // final merge is disabled. Final output file would not be populated in this case.
    assertTrue(sorter.finalOutputFile == null);
    TezCounter numShuffleChunks = outputContext.getCounters().findCounter(TaskCounter.SHUFFLE_CHUNK_COUNT);
//    assertTrue(sorter.getNumSpills() == numShuffleChunks.getValue());
    conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, true);

  }
 

import org.apache.tez.common.counters.TezCounter; //导入依赖的package包/类
@Test
public void testExceedsKVWithPipelinedShuffle() throws IOException {
  this.numOutputs = 1;
  this.initialAvailableMem = 1 *1024 * 1024;
  conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, false);
  conf.setInt(TezRuntimeConfiguration
      .TEZ_RUNTIME_PIPELINED_SORTER_MIN_BLOCK_SIZE_IN_MB, 1);
  PipelinedSorter sorter = new PipelinedSorter(this.outputContext, conf, numOutputs,
      initialAvailableMem);

  writeData(sorter, 5, 1<<20);

  // final merge is disabled. Final output file would not be populated in this case.
  assertTrue(sorter.finalOutputFile == null);
  TezCounter numShuffleChunks = outputContext.getCounters().findCounter(TaskCounter.SHUFFLE_CHUNK_COUNT);
  assertTrue(sorter.getNumSpills() == numShuffleChunks.getValue());
  conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, true);
}
 

import org.apache.tez.common.counters.TezCounter; //导入依赖的package包/类
@SuppressWarnings("unchecked")
private ValuesIterator createEmptyIterator(boolean inMemory)
    throws IOException, InterruptedException {
  if (!inMemory) {
    streamPaths = new Path[0];
    //This will return EmptyIterator
    rawKeyValueIterator =
        TezMerger.merge(conf, fs, keyClass, valClass, null,
            false, -1, 1024, streamPaths, false, mergeFactor, tmpDir, comparator,
            new ProgressReporter(), null, null, null, null);
  } else {
    List<TezMerger.Segment> segments = Lists.newLinkedList();
    //This will return EmptyIterator
    rawKeyValueIterator =
        TezMerger.merge(conf, fs, keyClass, valClass, segments, mergeFactor, tmpDir,
            comparator, new ProgressReporter(), new GenericCounter("readsCounter", "y"),
            new GenericCounter("writesCounter", "y1"),
            new GenericCounter("bytesReadCounter", "y2"), new Progress());
  }
  return new ValuesIterator(rawKeyValueIterator, comparator,
      keyClass, valClass, conf, (TezCounter) new GenericCounter("inputKeyCounter", "y3"),
      (TezCounter) new GenericCounter("inputValueCounter", "y4"));
}