Java MapTaskCompletionEventsUpdate类代码示例

import org.apache.hadoop.mapred.MapTaskCompletionEventsUpdate; //导入依赖的package包/类
private MapTaskCompletionEventsUpdate getMockedCompletionEventsUpdate(
    int startIdx, int numEvents) {
  ArrayList<TaskCompletionEvent> tceList =
      new ArrayList<TaskCompletionEvent>(numEvents);
  for (int i = 0; i < numEvents; ++i) {
    int eventIdx = startIdx + i;
    TaskCompletionEvent tce = new TaskCompletionEvent(eventIdx,
        new TaskAttemptID("12345", 1, TaskType.MAP, eventIdx, 0),
        eventIdx, true, TaskCompletionEvent.Status.SUCCEEDED,
        "http://somehost:8888");
    tceList.add(tce);
  }
  TaskCompletionEvent[] events = {};
  return new MapTaskCompletionEventsUpdate(tceList.toArray(events), false);
}
 

import org.apache.hadoop.mapred.MapTaskCompletionEventsUpdate; //导入依赖的package包/类
public MapTaskCompletionEventsUpdate getMapTaskCompletionEventsUpdates(
    int index, JobID jobId, int max) throws IOException {
  throw new UnsupportedOperationException();
}
 

import org.apache.hadoop.mapred.MapTaskCompletionEventsUpdate; //导入依赖的package包/类
/** 
 * Queries the {@link TaskTracker} for a set of map-completion events 
 * from a given event ID.
 * @throws IOException
 */  
private int getMapCompletionEvents() throws IOException {
  
  int numNewMaps = 0;
  
  MapTaskCompletionEventsUpdate update = 
    umbilical.getMapCompletionEvents((org.apache.hadoop.mapred.JobID)
                                     reduce.getJobID(), 
                                     fromEventId, 
                                     MAX_EVENTS_TO_FETCH,
                                     (org.apache.hadoop.mapred.TaskAttemptID)
                                       reduce);
  TaskCompletionEvent events[] = update.getMapTaskCompletionEvents();
  LOG.debug("Got " + events.length + " map completion events from " + 
           fromEventId);
    
  // Check if the reset is required.
  // Since there is no ordering of the task completion events at the 
  // reducer, the only option to sync with the new jobtracker is to reset 
  // the events index
  if (update.shouldReset()) {
    fromEventId = 0;
    scheduler.resetKnownMaps();
  }
  
  // Update the last seen event ID
  fromEventId += events.length;
  
  // Process the TaskCompletionEvents:
  // 1. Save the SUCCEEDED maps in knownOutputs to fetch the outputs.
  // 2. Save the OBSOLETE/FAILED/KILLED maps in obsoleteOutputs to stop 
  //    fetching from those maps.
  // 3. Remove TIPFAILED maps from neededOutputs since we don't need their
  //    outputs at all.
  for (TaskCompletionEvent event : events) {
    switch (event.getTaskStatus()) {
      case SUCCEEDED:
        URI u = getBaseURI(event.getTaskTrackerHttp());
        scheduler.addKnownMapOutput(u.getHost() + ":" + u.getPort(),
                                    u.toString(),
                                    event.getTaskAttemptId());
        numNewMaps ++;
        int duration = event.getTaskRunTime();
        if (duration > maxMapRuntime) {
          maxMapRuntime = duration;
          scheduler.informMaxMapRunTime(maxMapRuntime);
        }
        break;
      case FAILED:
      case KILLED:
      case OBSOLETE:
        scheduler.obsoleteMapOutput(event.getTaskAttemptId());
        LOG.info("Ignoring obsolete output of " + event.getTaskStatus() + 
                 " map-task: '" + event.getTaskAttemptId() + "'");
        break;
      case TIPFAILED:
        scheduler.tipFailed(event.getTaskAttemptId().getTaskID());
        LOG.info("Ignoring output of failed map TIP: '" +  
             event.getTaskAttemptId() + "'");
        break;
    }
  }
  return numNewMaps;
}