Java MRReader类代码示例

import org.apache.tez.mapreduce.lib.MRReader; //导入依赖的package包/类
@Override
public void attachInputs(Map<String, LogicalInput> inputs,
        Configuration conf)
        throws ExecException {
    this.conf = conf;
    LogicalInput logInput = inputs.get(inputKey);
    if (logInput == null || !(logInput instanceof MRInput)) {
        throw new ExecException("POSimpleTezLoad only accepts MRInputs");
    }
    input = (MRInput) logInput;
    try {
        reader = input.getReader();
        // Set split index, MergeCoGroup need it. And this input is the only input of the
        // MergeCoGroup vertex.
        if (reader instanceof MRReader) {
            int splitIndex = ((PigSplit)((MRReader)reader).getSplit()).getSplitIndex();
            PigMapReduce.sJobContext.getConfiguration().setInt(PigImplConstants.PIG_SPLIT_INDEX, splitIndex);
        }
    } catch (IOException e) {
        throw new ExecException(e);
    }
}
 

import org.apache.tez.mapreduce.lib.MRReader; //导入依赖的package包/类
public Collection<KeyValueReader> getKeyValueReaders() throws InterruptedException, IOException {
  lock.lock();
  try {
    while (eventCount.get() != getNumPhysicalInputs()) {
      condition.await();
    }
  } finally {
    lock.unlock();
  }
  return Collections
      .unmodifiableCollection(Lists.transform(readers, new Function<MRReader, KeyValueReader>() {
        @Override
        public KeyValueReader apply(MRReader input) {
          return input;
        }
      }));
}
 

import org.apache.tez.mapreduce.lib.MRReader; //导入依赖的package包/类
@Override
public void handleEvents(List<Event> inputEvents) throws Exception {
  lock.lock();
  try {
    Preconditions.checkState(eventCount.get() + inputEvents.size() <= getNumPhysicalInputs(),
        "Unexpected event. All physical sources already initialized");
    for (Event event : inputEvents) {
      MRReader reader = initFromEvent((RootInputDataInformationEvent) event);
      readers.add(reader);
      if (eventCount.incrementAndGet() == getNumPhysicalInputs()) {
        getContext().inputIsReady();
        condition.signal();
      }
    }
  } finally {
    lock.unlock();
  }
}
 

import org.apache.tez.mapreduce.lib.MRReader; //导入依赖的package包/类
@Override
public void handleEvents(List<Event> inputEvents) throws Exception {
  lock.lock();
  try {
    if (getNumPhysicalInputs() == 0) {
      throw new IllegalStateException(
          "Unexpected event. MultiMRInput has been setup to receive 0 events");
    }
    Preconditions.checkState(eventCount.get() + inputEvents.size() <= getNumPhysicalInputs(),
        "Unexpected event. All physical sources already initialized");
    for (Event event : inputEvents) {
      MRReader reader = initFromEvent((InputDataInformationEvent) event);
      readers.add(reader);
      if (eventCount.incrementAndGet() == getNumPhysicalInputs()) {
        getContext().inputIsReady();
        condition.signal();
      }
    }
  } finally {
    lock.unlock();
  }
}
 

import org.apache.tez.mapreduce.lib.MRReader; //导入依赖的package包/类
private MRReader initFromEvent(RootInputDataInformationEvent event) throws IOException {
  Preconditions.checkState(event != null, "Event must be specified");
  LOG.info("Initializing Reader: " + eventCount.get());
  MRSplitProto splitProto = MRSplitProto.parseFrom(event.getUserPayload());
  Object split = null;
  MRReader reader = null;
  JobConf localJobConf = new JobConf(jobConf);
  if (useNewApi) {
    split = MRInputUtils.getNewSplitDetailsFromEvent(splitProto, localJobConf);
    reader = new MRReaderMapReduce(localJobConf, (org.apache.hadoop.mapreduce.InputSplit) split,
        getContext().getCounters(), inputRecordCounter, getContext().getApplicationId()
        .getClusterTimestamp(), getContext().getTaskVertexIndex(), getContext()
        .getApplicationId().getId(), getContext().getTaskIndex(), getContext()
        .getTaskAttemptNumber());
    LOG.info("Split Details -> SplitClass: " + split.getClass().getName() + ", NewSplit: "
        + split);

  } else {
    split = MRInputUtils.getOldSplitDetailsFromEvent(splitProto, localJobConf);
    reader = new MRReaderMapred(localJobConf, (org.apache.hadoop.mapred.InputSplit) split,
        getContext().getCounters(), inputRecordCounter);
    LOG.info("Split Details -> SplitClass: " + split.getClass().getName() + ", OldSplit: "
        + split);
  }
  LOG.info("Initialized RecordReader from event");
  return reader;
}
 

import org.apache.tez.mapreduce.lib.MRReader; //导入依赖的package包/类
@Override
public List<Event> close() throws Exception {
  for (MRReader reader : readers) {
    reader.close();
  }
  return null;
}
 

import org.apache.tez.mapreduce.lib.MRReader; //导入依赖的package包/类
@Override
public List<Event> close() throws Exception {
  for (MRReader reader : readers) {
    reader.close();
  }
  long inputRecords = getContext().getCounters()
      .findCounter(TaskCounter.INPUT_RECORDS_PROCESSED).getValue();
  getContext().getStatisticsReporter().reportItemsProcessed(inputRecords);

  return null;
}