Java Phase类代码示例

import org.apache.hadoop.mapreduce.v2.api.records.Phase; //导入依赖的package包/类
private void updateStatus(MRApp app, TaskAttempt attempt, Phase phase) {
  TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus();
  status.counters = new Counters();
  status.fetchFailedMaps = new ArrayList<TaskAttemptId>();
  status.id = attempt.getID();
  status.mapFinishTime = 0;
  status.phase = phase;
  status.progress = 0.5f;
  status.shuffleFinishTime = 0;
  status.sortFinishTime = 0;
  status.stateString = "OK";
  status.taskState = attempt.getState();
  TaskAttemptStatusUpdateEvent event = new TaskAttemptStatusUpdateEvent(attempt.getID(),
      status);
  app.getContext().getEventHandler().handle(event);
}
 

import org.apache.hadoop.mapreduce.v2.api.records.Phase; //导入依赖的package包/类
public static Phase toYarn(org.apache.hadoop.mapred.TaskStatus.Phase phase) {
  switch (phase) {
  case STARTING:
    return Phase.STARTING;
  case MAP:
    return Phase.MAP;
  case SHUFFLE:
    return Phase.SHUFFLE;
  case SORT:
    return Phase.SORT;
  case REDUCE:
    return Phase.REDUCE;
  case CLEANUP:
    return Phase.CLEANUP;
  }
  throw new YarnRuntimeException("Unrecognized Phase: " + phase);
}
 

import org.apache.hadoop.mapreduce.v2.api.records.Phase; //导入依赖的package包/类
/**
 * test some methods of CompletedTaskAttempt
 */
@Test (timeout=5000)
public void testCompletedTaskAttempt(){
  
  TaskAttemptInfo attemptInfo= mock(TaskAttemptInfo.class);
  when(attemptInfo.getRackname()).thenReturn("Rackname");
  when(attemptInfo.getShuffleFinishTime()).thenReturn(11L);
  when(attemptInfo.getSortFinishTime()).thenReturn(12L);
  when(attemptInfo.getShufflePort()).thenReturn(10);
  
  JobID jobId= new JobID("12345",0);
  TaskID taskId =new TaskID(jobId,TaskType.REDUCE, 0);
  TaskAttemptID taskAttemptId= new TaskAttemptID(taskId, 0);
  when(attemptInfo.getAttemptId()).thenReturn(taskAttemptId);
  
  
  CompletedTaskAttempt taskAttemt= new CompletedTaskAttempt(null,attemptInfo);
  assertEquals( "Rackname",   taskAttemt.getNodeRackName());
  assertEquals( Phase.CLEANUP,   taskAttemt.getPhase());
  assertTrue(  taskAttemt.isFinished());
  assertEquals( 11L,   taskAttemt.getShuffleFinishTime());
  assertEquals( 12L,   taskAttemt.getSortFinishTime());
  assertEquals( 10,   taskAttemt.getShufflePort());
}
 

import org.apache.hadoop.mapreduce.v2.api.records.Phase; //导入依赖的package包/类
public static Phase toYarn(org.apache.hadoop.mapred.TaskStatus.Phase phase) {
  switch (phase) {
  case STARTING:
    return Phase.STARTING;
  case MAP:
    return Phase.MAP;
  case SHUFFLE:
    return Phase.SHUFFLE;
  case SORT:
    return Phase.SORT;
  case REDUCE:
    return Phase.REDUCE;
  case CLEANUP:
    return Phase.CLEANUP;
  default:
    break;
  }
  throw new YarnRuntimeException("Unrecognized Phase: " + phase);
}
 

import org.apache.hadoop.mapreduce.v2.api.records.Phase; //导入依赖的package包/类
@Override
public Phase getPhase() {
  readLock.lock();
  try {
    return reportedStatus.phase;
  } finally {
    readLock.unlock();
  }
}
 

import org.apache.hadoop.mapreduce.v2.api.records.Phase; //导入依赖的package包/类
private void initTaskAttemptStatus(TaskAttemptStatus result) {
  result.progress = 0.0f;
  result.phase = Phase.STARTING;
  result.stateString = "NEW";
  result.taskState = TaskAttemptState.NEW;
  Counters counters = EMPTY_COUNTERS;
  result.counters = counters;
}
 

import org.apache.hadoop.mapreduce.v2.api.records.Phase; //导入依赖的package包/类
@Override
public void transition(JobImpl job, JobEvent event) {
  //get number of shuffling reduces
  int shufflingReduceTasks = 0;
  for (TaskId taskId : job.reduceTasks) {
    Task task = job.tasks.get(taskId);
    if (TaskState.RUNNING.equals(task.getState())) {
      for(TaskAttempt attempt : task.getAttempts().values()) {
        if(attempt.getPhase() == Phase.SHUFFLE) {
          shufflingReduceTasks++;
          break;
        }
      }
    }
  }

  JobTaskAttemptFetchFailureEvent fetchfailureEvent = 
    (JobTaskAttemptFetchFailureEvent) event;
  for (org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId mapId : 
        fetchfailureEvent.getMaps()) {
    Integer fetchFailures = job.fetchFailuresMapping.get(mapId);
    fetchFailures = (fetchFailures == null) ? 1 : (fetchFailures+1);
    job.fetchFailuresMapping.put(mapId, fetchFailures);
    
    float failureRate = shufflingReduceTasks == 0 ? 1.0f : 
      (float) fetchFailures / shufflingReduceTasks;
    // declare faulty if fetch-failures >= max-allowed-failures
    if (fetchFailures >= job.getMaxFetchFailuresNotifications()
        && failureRate >= job.getMaxAllowedFetchFailuresFraction()) {
      LOG.info("Too many fetch-failures for output of task attempt: " + 
          mapId + " ... raising fetch failure to map");
      job.eventHandler.handle(new TaskAttemptEvent(mapId, 
          TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE));
      job.fetchFailuresMapping.remove(mapId);
    }
  }
}
 

import org.apache.hadoop.mapreduce.v2.api.records.Phase; //导入依赖的package包/类
@Override
public Phase getPhase() {
  TaskAttemptReportProtoOrBuilder p = viaProto ? proto : builder;
  if (!p.hasPhase()) {
    return null;
  }
  return convertFromProtoFormat(p.getPhase());
}
 

import org.apache.hadoop.mapreduce.v2.api.records.Phase; //导入依赖的package包/类
@Override
public void setPhase(Phase phase) {
  maybeInitBuilder();
  if (phase == null) {
    builder.clearPhase();
    return;
  }
  builder.setPhase(convertToProtoFormat(phase));
}
 

import org.apache.hadoop.mapreduce.v2.api.records.Phase; //导入依赖的package包/类
@Override
public void transition(JobImpl job, JobEvent event) {
  //get number of shuffling reduces
  int shufflingReduceTasks = 0;
  for (TaskId taskId : job.reduceTasks) {
    Task task = job.tasks.get(taskId);
    if (TaskState.RUNNING.equals(task.getState())) {
      for(TaskAttempt attempt : task.getAttempts().values()) {
        if(attempt.getPhase() == Phase.SHUFFLE) {
          shufflingReduceTasks++;
          break;
        }
      }
    }
  }

  JobTaskAttemptFetchFailureEvent fetchfailureEvent = 
    (JobTaskAttemptFetchFailureEvent) event;
  for (org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId mapId : 
        fetchfailureEvent.getMaps()) {
    Integer fetchFailures = job.fetchFailuresMapping.get(mapId);
    fetchFailures = (fetchFailures == null) ? 1 : (fetchFailures+1);
    job.fetchFailuresMapping.put(mapId, fetchFailures);
    
    float failureRate = shufflingReduceTasks == 0 ? 1.0f : 
      (float) fetchFailures / shufflingReduceTasks;
    // declare faulty if fetch-failures >= max-allowed-failures
    if (fetchFailures >= job.getMaxFetchFailuresNotifications()
        && failureRate >= job.getMaxAllowedFetchFailuresFraction()) {
      LOG.info("Too many fetch-failures for output of task attempt: " + 
          mapId + " ... raising fetch failure to map");
      job.eventHandler.handle(new TaskAttemptTooManyFetchFailureEvent(mapId,
          fetchfailureEvent.getReduce(), fetchfailureEvent.getHost()));
      job.fetchFailuresMapping.remove(mapId);
    }
  }
}