Java TaskAttemptState.SUCCEEDED属性代码示例

public static TaskAttemptState toYarn(
    org.apache.hadoop.mapred.TaskStatus.State state) {
  switch (state) {
  case COMMIT_PENDING:
    return TaskAttemptState.COMMIT_PENDING;
  case FAILED:
  case FAILED_UNCLEAN:
    return TaskAttemptState.FAILED;
  case KILLED:
  case KILLED_UNCLEAN:
    return TaskAttemptState.KILLED;
  case RUNNING:
    return TaskAttemptState.RUNNING;
  case SUCCEEDED:
    return TaskAttemptState.SUCCEEDED;
  case UNASSIGNED:
    return TaskAttemptState.STARTING;
  default:
    throw new YarnRuntimeException("Unrecognized State: " + state);
  }
}
 

private TaskAttempt getSuccessfulAttempt(Task task) {
  for (TaskAttempt attempt : task.getAttempts().values()) {
    if (attempt.getState() == TaskAttemptState.SUCCEEDED) {
      return attempt;
    }
  }
  return null;
}
 

private static TaskAttemptState getExternalState(
    TaskAttemptStateInternal smState) {
  switch (smState) {
  case ASSIGNED:
  case UNASSIGNED:
    return TaskAttemptState.STARTING;
  case COMMIT_PENDING:
    return TaskAttemptState.COMMIT_PENDING;
  case FAILED:
    return TaskAttemptState.FAILED;
  case KILLED:
    return TaskAttemptState.KILLED;
    // All CLEANUP states considered as RUNNING since events have not gone out
    // to the Task yet. May be possible to consider them as a Finished state.
  case FAIL_CONTAINER_CLEANUP:
  case FAIL_TASK_CLEANUP:
  case KILL_CONTAINER_CLEANUP:
  case KILL_TASK_CLEANUP:
  case SUCCESS_CONTAINER_CLEANUP:
  case RUNNING:
    return TaskAttemptState.RUNNING;
  case NEW:
    return TaskAttemptState.NEW;
  case SUCCEEDED:
    return TaskAttemptState.SUCCEEDED;
  default:
    throw new YarnRuntimeException("Attempt to convert invalid "
        + "stateMachineTaskAttemptState to externalTaskAttemptState: "
        + smState);
  }
}
 

@Override
public boolean isFinished() {
  for (TaskAttempt attempt : attempts.values()) {
    if (attempt.getState() == TaskAttemptState.SUCCEEDED) {
      return true;
    }
  }

  return false;
}
 

protected static TaskAttemptState getExternalState(
    TaskAttemptStateInternal smState) {
  switch (smState) {
  case ASSIGNED:
  case UNASSIGNED:
    return TaskAttemptState.STARTING;
  case COMMIT_PENDING:
    return TaskAttemptState.COMMIT_PENDING;
  case FAIL_CONTAINER_CLEANUP:
  case FAIL_TASK_CLEANUP:
  case FAIL_FINISHING_CONTAINER:
  case FAILED:
    return TaskAttemptState.FAILED;
  case KILL_CONTAINER_CLEANUP:
  case KILL_TASK_CLEANUP:
  case KILLED:
    return TaskAttemptState.KILLED;
  case RUNNING:
    return TaskAttemptState.RUNNING;
  case NEW:
    return TaskAttemptState.NEW;
  case SUCCESS_CONTAINER_CLEANUP:
  case SUCCESS_FINISHING_CONTAINER:
  case SUCCEEDED:
    return TaskAttemptState.SUCCEEDED;
  default:
    throw new YarnRuntimeException("Attempt to convert invalid "
        + "stateMachineTaskAttemptState to externalTaskAttemptState: "
        + smState);
  }
}
 

@Override
public void updateAttempt(TaskAttemptStatus status, long timestamp) {

  TaskAttemptId attemptID = status.id;
  TaskId taskID = attemptID.getTaskId();
  JobId jobID = taskID.getJobId();
  Job job = context.getJob(jobID);

  if (job == null) {
    return;
  }

  Task task = job.getTask(taskID);

  if (task == null) {
    return;
  }

  Long boxedStart = startTimes.get(attemptID);
  long start = boxedStart == null ? Long.MIN_VALUE : boxedStart;
  
  TaskAttempt taskAttempt = task.getAttempt(attemptID);

  if (taskAttempt.getState() == TaskAttemptState.SUCCEEDED) {
    boolean isNew = false;
    // is this  a new success?
    synchronized (doneTasks) {
      if (!doneTasks.contains(task)) {
        doneTasks.add(task);
        isNew = true;
      }
    }

    // It's a new completion
    // Note that if a task completes twice [because of a previous speculation
    //  and a race, or a success followed by loss of the machine with the
    //  local data] we only count the first one.
    if (isNew) {
      long finish = timestamp;
      if (start > 1L && finish > 1L && start <= finish) {
        long duration = finish - start;

        DataStatistics statistics
        = dataStatisticsForTask(taskID);

        if (statistics != null) {
          statistics.add(duration);
        }
      }
    }
  }
}
 

@Override
public TaskAttemptState getState() {
  if (overridingState != null) {
    return overridingState;
  }
  TaskAttemptState result
      = getProgress() < 1.0F ? TaskAttemptState.RUNNING : TaskAttemptState.SUCCEEDED;

  if (result == TaskAttemptState.SUCCEEDED) {
    overridingState = TaskAttemptState.SUCCEEDED;

    System.out.println("MyTaskAttemptImpl.getState() -- attempt " + myAttemptID + " finished.");

    slotsInUse.addAndGet(- taskTypeSlots(myAttemptID.getTaskId().getTaskType()));

    (myAttemptID.getTaskId().getTaskType() == TaskType.MAP
        ? completedMaps : completedReduces).getAndIncrement();

    // check for a spectacularly successful speculation
    TaskId taskID = myAttemptID.getTaskId();

    Task task = myJob.getTask(taskID);

    for (TaskAttempt otherAttempt : task.getAttempts().values()) {
      if (otherAttempt != this
          && otherAttempt.getState() == TaskAttemptState.RUNNING) {
        // we had two instances running.  Try to determine how much
        //  we might have saved by speculation
        if (getID().getId() > otherAttempt.getID().getId()) {
          // the speculation won
          successfulSpeculations.getAndIncrement();
          float hisProgress = otherAttempt.getProgress();
          long hisStartTime = ((MyTaskAttemptImpl)otherAttempt).startMockTime;
          System.out.println("TLTRE:A speculation finished at time "
              + clock.getTime()
              + ".  The stalled attempt is at " + (hisProgress * 100.0)
              + "% progress, and it started at "
              + hisStartTime + ", which is "
              + (clock.getTime() - hisStartTime) + " ago.");
          long originalTaskEndEstimate
              = (hisStartTime
                  + estimator.estimatedRuntime(otherAttempt.getID()));
          System.out.println(
              "TLTRE: We would have expected the original attempt to take "
              + estimator.estimatedRuntime(otherAttempt.getID())
              + ", finishing at " + originalTaskEndEstimate);
          long estimatedSavings = originalTaskEndEstimate - clock.getTime();
          taskTimeSavedBySpeculation.addAndGet(estimatedSavings);
          System.out.println("TLTRE: The task is " + task.getID());
          slotsInUse.addAndGet(- taskTypeSlots(myAttemptID.getTaskId().getTaskType()));
          ((MyTaskAttemptImpl)otherAttempt).overridingState
              = TaskAttemptState.KILLED;
        } else {
          System.out.println(
              "TLTRE: The normal attempt beat the speculation in "
              + task.getID());
        }
      }
    }
  }

  return result;
}
 

/**
 * Go through a job and update the member variables with counts for
 * information to output in the page.
 *
 * @param job
 *          the job to get counts for.
 */
private void countTasksAndAttempts(Job job) {
  numReduces = 0;
  numMaps = 0;
  final Map<TaskId, Task> tasks = job.getTasks();
  if (tasks == null) {
    return;
  }
  for (Task task : tasks.values()) {
    // Attempts counts
    Map<TaskAttemptId, TaskAttempt> attempts = task.getAttempts();
    int successful, failed, killed;
    for (TaskAttempt attempt : attempts.values()) {

      successful = 0;
      failed = 0;
      killed = 0;
      if (TaskAttemptStateUI.NEW.correspondsTo(attempt.getState())) {
        // Do Nothing
      } else if (TaskAttemptStateUI.RUNNING.correspondsTo(attempt.getState())) {
        // Do Nothing
      } else if (TaskAttemptStateUI.SUCCESSFUL.correspondsTo(attempt
          .getState())) {
        ++successful;
      } else if (TaskAttemptStateUI.FAILED.correspondsTo(attempt.getState())) {
        ++failed;
      } else if (TaskAttemptStateUI.KILLED.correspondsTo(attempt.getState())) {
        ++killed;
      }

      switch (task.getType()) {
      case MAP:
        successfulMapAttempts += successful;
        failedMapAttempts += failed;
        killedMapAttempts += killed;
        if (attempt.getState() == TaskAttemptState.SUCCEEDED) {
          numMaps++;
          avgMapTime += (attempt.getFinishTime() - attempt.getLaunchTime());
        }
        break;
      case REDUCE:
        successfulReduceAttempts += successful;
        failedReduceAttempts += failed;
        killedReduceAttempts += killed;
        if (attempt.getState() == TaskAttemptState.SUCCEEDED) {
          numReduces++;
          avgShuffleTime += (attempt.getShuffleFinishTime() - attempt
              .getLaunchTime());
          avgMergeTime += attempt.getSortFinishTime()
              - attempt.getShuffleFinishTime();
          avgReduceTime += (attempt.getFinishTime() - attempt
              .getSortFinishTime());
        }
        break;
      }
    }
  }

  if (numMaps > 0) {
    avgMapTime = avgMapTime / numMaps;
  }

  if (numReduces > 0) {
    avgReduceTime = avgReduceTime / numReduces;
    avgShuffleTime = avgShuffleTime / numReduces;
    avgMergeTime = avgMergeTime / numReduces;
  }
}
 

/**
 * test AttemptsBlock's rendering.
 */
@Test
public void testAttemptsBlock() {
  AppContext ctx = mock(AppContext.class);
  AppForTest app = new AppForTest(ctx);

  Task task = getTask(0);
  Map<TaskAttemptId, TaskAttempt> attempts = new HashMap<TaskAttemptId, TaskAttempt>();
  TaskAttempt attempt = mock(TaskAttempt.class);
  TaskAttemptId taId = new TaskAttemptIdPBImpl();
  taId.setId(0);
  taId.setTaskId(task.getID());
  when(attempt.getID()).thenReturn(taId);
  when(attempt.getNodeHttpAddress()).thenReturn("Node address");

  ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 5);
  ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.newInstance(appId, 1);

  ContainerId containerId = ContainerIdPBImpl.newContainerId(appAttemptId, 1);
  when(attempt.getAssignedContainerID()).thenReturn(containerId);

  when(attempt.getAssignedContainerMgrAddress()).thenReturn(
          "assignedContainerMgrAddress");
  when(attempt.getNodeRackName()).thenReturn("nodeRackName");

  final long taStartTime = 100002L;
  final long taFinishTime = 100012L;
  final long taShuffleFinishTime = 100010L;
  final long taSortFinishTime = 100011L;
  final TaskAttemptState taState = TaskAttemptState.SUCCEEDED;

  when(attempt.getLaunchTime()).thenReturn(taStartTime);
  when(attempt.getFinishTime()).thenReturn(taFinishTime);
  when(attempt.getShuffleFinishTime()).thenReturn(taShuffleFinishTime);
  when(attempt.getSortFinishTime()).thenReturn(taSortFinishTime);
  when(attempt.getState()).thenReturn(taState);

  TaskAttemptReport taReport = mock(TaskAttemptReport.class);
  when(taReport.getStartTime()).thenReturn(taStartTime);
  when(taReport.getFinishTime()).thenReturn(taFinishTime);
  when(taReport.getShuffleFinishTime()).thenReturn(taShuffleFinishTime);
  when(taReport.getSortFinishTime()).thenReturn(taSortFinishTime);
  when(taReport.getContainerId()).thenReturn(containerId);
  when(taReport.getProgress()).thenReturn(1.0f);
  when(taReport.getStateString()).thenReturn("Processed 128/128 records <p> \n");
  when(taReport.getTaskAttemptState()).thenReturn(taState);
  when(taReport.getDiagnosticInfo()).thenReturn("");

  when(attempt.getReport()).thenReturn(taReport);

  attempts.put(taId, attempt);
  when(task.getAttempts()).thenReturn(attempts);

  app.setTask(task);
  Job job = mock(Job.class);
  when(job.getUserName()).thenReturn("User");
  app.setJob(job);

  AttemptsBlockForTest block = new AttemptsBlockForTest(app);
  block.addParameter(AMParams.TASK_TYPE, "r");

  PrintWriter pWriter = new PrintWriter(data);
  Block html = new BlockForTest(new HtmlBlockForTest(), pWriter, 0, false);

  block.render(html);
  pWriter.flush();
  // should be printed information about attempts
  assertTrue(data.toString().contains("0 attempt_0_0001_r_000000_0"));
  assertTrue(data.toString().contains("SUCCEEDED"));
  assertFalse(data.toString().contains("Processed 128/128 records <p> \n"));
  assertTrue(data.toString().contains("Processed 128\\/128 records &lt;p&gt; \\n"));
  assertTrue(data.toString().contains(
          "_0005_01_000001:attempt_0_0001_r_000000_0:User:"));
  assertTrue(data.toString().contains("100002"));
  assertTrue(data.toString().contains("100010"));
  assertTrue(data.toString().contains("100011"));
  assertTrue(data.toString().contains("100012"));
}
 

/**
 * test AttemptsBlock's rendering.
 */
@Test
public void testAttemptsBlock() {
  AppContext ctx = mock(AppContext.class);
  AppForTest app = new AppForTest(ctx);

  JobId jobId = new JobIdPBImpl();
  jobId.setId(0);
  jobId.setAppId(ApplicationIdPBImpl.newInstance(0,1));

  TaskId taskId = new TaskIdPBImpl();
  taskId.setId(0);
  taskId.setTaskType(TaskType.REDUCE);
  taskId.setJobId(jobId);
  Task task = mock(Task.class);
  when(task.getID()).thenReturn(taskId);
  TaskReport report = mock(TaskReport.class);

  when(task.getReport()).thenReturn(report);
  when(task.getType()).thenReturn(TaskType.REDUCE);

  Map<TaskId, Task> tasks =
      new HashMap<TaskId, Task>();
  Map<TaskAttemptId, TaskAttempt> attempts =
      new HashMap<TaskAttemptId, TaskAttempt>();
  TaskAttempt attempt = mock(TaskAttempt.class);
  TaskAttemptId taId = new TaskAttemptIdPBImpl();
  taId.setId(0);
  taId.setTaskId(task.getID());
  when(attempt.getID()).thenReturn(taId);

  final TaskAttemptState taState = TaskAttemptState.SUCCEEDED;
  when(attempt.getState()).thenReturn(taState);
  TaskAttemptReport taReport = mock(TaskAttemptReport.class);
  when(taReport.getTaskAttemptState()).thenReturn(taState);
  when(attempt.getReport()).thenReturn(taReport);
  attempts.put(taId, attempt);
  tasks.put(taskId, task);
  when(task.getAttempts()).thenReturn(attempts);

  app.setTask(task);
  Job job = mock(Job.class);
  when(job.getTasks(TaskType.REDUCE)).thenReturn(tasks);
  app.setJob(job);

  AttemptsBlockForTest block = new AttemptsBlockForTest(app,
      new Configuration());
  block.addParameter(AMParams.TASK_TYPE, "r");
  block.addParameter(AMParams.ATTEMPT_STATE, "SUCCESSFUL");

  PrintWriter pWriter = new PrintWriter(data);
  Block html = new BlockForTest(new HtmlBlockForTest(), pWriter, 0, false);

  block.render(html);
  pWriter.flush();
  assertTrue(data.toString().contains(
      "<a href='" + block.url("task",task.getID().toString()) +"'>"
      +"attempt_0_0001_r_000000_0</a>"));
}
 

/**
 * test AttemptsBlock's rendering.
 */
@Test
public void testAttemptsBlock() {
  AppContext ctx = mock(AppContext.class);
  AppForTest app = new AppForTest(ctx);

  Task task = getTask(0);
  Map<TaskAttemptId, TaskAttempt> attempts = new HashMap<TaskAttemptId, TaskAttempt>();
  TaskAttempt attempt = mock(TaskAttempt.class);
  TaskAttemptId taId = new TaskAttemptIdPBImpl();
  taId.setId(0);
  taId.setTaskId(task.getID());
  when(attempt.getID()).thenReturn(taId);
  when(attempt.getNodeHttpAddress()).thenReturn("Node address");

  ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 5);
  ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.newInstance(appId, 1);

  ContainerId containerId = ContainerIdPBImpl.newContainerId(appAttemptId, 1);
  when(attempt.getAssignedContainerID()).thenReturn(containerId);

  when(attempt.getAssignedContainerMgrAddress()).thenReturn(
          "assignedContainerMgrAddress");
  when(attempt.getNodeRackName()).thenReturn("nodeRackName");

  final long taStartTime = 100002L;
  final long taFinishTime = 100012L;
  final long taShuffleFinishTime = 100010L;
  final long taSortFinishTime = 100011L;
  final TaskAttemptState taState = TaskAttemptState.SUCCEEDED;

  when(attempt.getLaunchTime()).thenReturn(taStartTime);
  when(attempt.getFinishTime()).thenReturn(taFinishTime);
  when(attempt.getShuffleFinishTime()).thenReturn(taShuffleFinishTime);
  when(attempt.getSortFinishTime()).thenReturn(taSortFinishTime);
  when(attempt.getState()).thenReturn(taState);

  TaskAttemptReport taReport = mock(TaskAttemptReport.class);
  when(taReport.getStartTime()).thenReturn(taStartTime);
  when(taReport.getFinishTime()).thenReturn(taFinishTime);
  when(taReport.getShuffleFinishTime()).thenReturn(taShuffleFinishTime);
  when(taReport.getSortFinishTime()).thenReturn(taSortFinishTime);
  when(taReport.getContainerId()).thenReturn(containerId);
  when(taReport.getProgress()).thenReturn(1.0f);
  when(taReport.getStateString()).thenReturn("Processed 128/128 records <p> \n");
  when(taReport.getTaskAttemptState()).thenReturn(taState);
  when(taReport.getDiagnosticInfo()).thenReturn("");

  when(attempt.getReport()).thenReturn(taReport);

  attempts.put(taId, attempt);
  when(task.getAttempts()).thenReturn(attempts);

  app.setTask(task);
  Job job = mock(Job.class);
  when(job.getUserName()).thenReturn("User");
  app.setJob(job);

  AttemptsBlockForTest block = new AttemptsBlockForTest(app);
  block.addParameter(AMParams.TASK_TYPE, "r");

  PrintWriter pWriter = new PrintWriter(data);
  Block html = new BlockForTest(new HtmlBlockForTest(), pWriter, 0, false);

  block.render(html);
  pWriter.flush();
  // should be printed information about attempts
  assertTrue(data.toString().contains("attempt_0_0001_r_000000_0"));
  assertTrue(data.toString().contains("SUCCEEDED"));
  assertFalse(data.toString().contains("Processed 128/128 records <p> \n"));
  assertTrue(data.toString().contains("Processed 128\\/128 records &lt;p&gt; \\n"));
  assertTrue(data.toString().contains(
          "_0005_01_000001:attempt_0_0001_r_000000_0:User:"));
  assertTrue(data.toString().contains("100002"));
  assertTrue(data.toString().contains("100010"));
  assertTrue(data.toString().contains("100011"));
  assertTrue(data.toString().contains("100012"));
}