Java JobInfo.getAllTasks方法代码示例

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo; //导入方法依赖的package包/类
private long computeFinishedMaps(JobInfo jobInfo, int numMaps,
    int numSuccessfulMaps) {
  if (numMaps == numSuccessfulMaps) {
    return jobInfo.getFinishedMaps();
  }

  long numFinishedMaps = 0;
  Map<org.apache.hadoop.mapreduce.TaskID, TaskInfo> taskInfos = jobInfo
      .getAllTasks();
  for (TaskInfo taskInfo : taskInfos.values()) {
    if (TaskState.SUCCEEDED.toString().equals(taskInfo.getTaskStatus())) {
      ++numFinishedMaps;
    }
  }
  return numFinishedMaps;
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo; //导入方法依赖的package包/类
/** Apply the filter (status) on the parsed job and generate summary */
public FilteredJob(JobInfo job, String status) {

  filter = status;
  
  Map<TaskID, JobHistoryParser.TaskInfo> tasks = job.getAllTasks();

  for (JobHistoryParser.TaskInfo task : tasks.values()) {
    Map<TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts =
      task.getAllTaskAttempts();
    for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) {
      if (attempt.getTaskStatus().equals(status)) {
        String hostname = attempt.getHostname();
        TaskID id = attempt.getAttemptId().getTaskID();

        Set<TaskID> set = badNodesToFilteredTasks.get(hostname);

        if (set == null) {
          set = new TreeSet<TaskID>();
          set.add(id);
          badNodesToFilteredTasks.put(hostname, set);
        }else{
          set.add(id);
        }
      }
    }
  }
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo; //导入方法依赖的package包/类
private Set<TimelineEntity> createTaskAndTaskAttemptEntities(JobInfo jobInfo) {
  Set<TimelineEntity> entities = new HashSet<>();
  Map<TaskID,TaskInfo> taskInfoMap = jobInfo.getAllTasks();
  LOG.info("job " + jobInfo.getJobId()+ " has " + taskInfoMap.size() +
      " tasks");
  for (TaskInfo taskInfo: taskInfoMap.values()) {
    TimelineEntity task = createTaskEntity(taskInfo);
    entities.add(task);
    // add the task attempts from this task
    Set<TimelineEntity> taskAttempts = createTaskAttemptEntities(taskInfo);
    entities.addAll(taskAttempts);
  }
  return entities;
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo; //导入方法依赖的package包/类
private void parsePreviousJobHistory() throws IOException {
  FSDataInputStream in = getPreviousJobHistoryStream(getConfig(),
      appAttemptID);
  JobHistoryParser parser = new JobHistoryParser(in);
  JobInfo jobInfo = parser.parse();
  Exception parseException = parser.getParseException();
  if (parseException != null) {
    LOG.info("Got an error parsing job-history file" +
        ", ignoring incomplete events.", parseException);
  }
  Map<org.apache.hadoop.mapreduce.TaskID, TaskInfo> taskInfos = jobInfo
      .getAllTasks();
  for (TaskInfo taskInfo : taskInfos.values()) {
    if (TaskState.SUCCEEDED.toString().equals(taskInfo.getTaskStatus())) {
      Iterator<Entry<TaskAttemptID, TaskAttemptInfo>> taskAttemptIterator =
          taskInfo.getAllTaskAttempts().entrySet().iterator();
      while (taskAttemptIterator.hasNext()) {
        Map.Entry<TaskAttemptID, TaskAttemptInfo> currentEntry = taskAttemptIterator.next();
        if (!jobInfo.getAllCompletedTaskAttempts().containsKey(currentEntry.getKey())) {
          taskAttemptIterator.remove();
        }
      }
      completedTasksFromPreviousRun
          .put(TypeConverter.toYarn(taskInfo.getTaskId()), taskInfo);
      LOG.info("Read from history task "
          + TypeConverter.toYarn(taskInfo.getTaskId()));
    }
  }
  LOG.info("Read completed tasks from history "
      + completedTasksFromPreviousRun.size());
  recoveredJobStartTime = jobInfo.getLaunchTime();

  // recover AMInfos
  List<JobHistoryParser.AMInfo> jhAmInfoList = jobInfo.getAMInfos();
  if (jhAmInfoList != null) {
    for (JobHistoryParser.AMInfo jhAmInfo : jhAmInfoList) {
      AMInfo amInfo = MRBuilderUtils.newAMInfo(jhAmInfo.getAppAttemptId(),
          jhAmInfo.getStartTime(), jhAmInfo.getContainerId(),
          jhAmInfo.getNodeManagerHost(), jhAmInfo.getNodeManagerPort(),
          jhAmInfo.getNodeManagerHttpPort());
      amInfos.add(amInfo);
    }
  }
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo; //导入方法依赖的package包/类
/** Create summary information for the parsed job */
public SummarizedJob(JobInfo job) {
  tasks = job.getAllTasks();

  for (JobHistoryParser.TaskInfo task : tasks.values()) {
    Map<TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = 
      task.getAllTaskAttempts();
    //allHosts.put(task.getHo(Keys.HOSTNAME), "");
    for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) {
      long startTime = attempt.getStartTime(); 
      long finishTime = attempt.getFinishTime();
      if (attempt.getTaskType().equals(TaskType.MAP)) {
        if (mapStarted== 0 || mapStarted > startTime) {
          mapStarted = startTime; 
        }
        if (mapFinished < finishTime) {
          mapFinished = finishTime; 
        }
        totalMaps++; 
        if (attempt.getTaskStatus().equals
            (TaskStatus.State.FAILED.toString())) {
          numFailedMaps++; 
        } else if (attempt.getTaskStatus().equals
            (TaskStatus.State.KILLED.toString())) {
          numKilledMaps++;
        }
      } else if (attempt.getTaskType().equals(TaskType.REDUCE)) {
        if (reduceStarted==0||reduceStarted > startTime) {
          reduceStarted = startTime; 
        }
        if (reduceFinished < finishTime) {
          reduceFinished = finishTime; 
        }
        totalReduces++; 
        if (attempt.getTaskStatus().equals
            (TaskStatus.State.FAILED.toString())) {
          numFailedReduces++; 
        } else if (attempt.getTaskStatus().equals
            (TaskStatus.State.KILLED.toString())) {
          numKilledReduces++;
        }
      } else if (attempt.getTaskType().equals(TaskType.JOB_CLEANUP)) {
        if (cleanupStarted==0||cleanupStarted > startTime) {
          cleanupStarted = startTime; 
        }
        if (cleanupFinished < finishTime) {
          cleanupFinished = finishTime; 
        }
        totalCleanups++; 
        if (attempt.getTaskStatus().equals
            (TaskStatus.State.SUCCEEDED.toString())) {
          numFinishedCleanups++; 
        } else if (attempt.getTaskStatus().equals
            (TaskStatus.State.FAILED.toString())) {
          numFailedCleanups++;
        } else if (attempt.getTaskStatus().equals
            (TaskStatus.State.KILLED.toString())) {
          numKilledCleanups++;
        }
      } else if (attempt.getTaskType().equals(TaskType.JOB_SETUP)) {
        if (setupStarted==0||setupStarted > startTime) {
          setupStarted = startTime; 
        }
        if (setupFinished < finishTime) {
          setupFinished = finishTime; 
        }
        totalSetups++; 
        if (attempt.getTaskStatus().equals
            (TaskStatus.State.SUCCEEDED.toString())) {
          numFinishedSetups++;
        } else if (attempt.getTaskStatus().equals
            (TaskStatus.State.FAILED.toString())) {
          numFailedSetups++;
        } else if (attempt.getTaskStatus().equals
            (TaskStatus.State.KILLED.toString())) {
          numKilledSetups++;
        }
      }
    }
  }
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo; //导入方法依赖的package包/类
/** Generate analysis information for the parsed job */
public AnalyzedJob (JobInfo job) {
  Map<TaskID, JobHistoryParser.TaskInfo> tasks = job.getAllTasks();
  int finishedMaps = (int) job.getFinishedMaps();
  int finishedReduces = (int) job.getFinishedReduces();
  mapTasks = 
    new JobHistoryParser.TaskAttemptInfo[finishedMaps]; 
  reduceTasks = 
    new JobHistoryParser.TaskAttemptInfo[finishedReduces]; 
  int mapIndex = 0 , reduceIndex=0; 
  avgMapTime = 0;
  avgReduceTime = 0;
  avgShuffleTime = 0;

  for (JobHistoryParser.TaskInfo task : tasks.values()) {
    Map<TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts =
      task.getAllTaskAttempts();
    for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) {
      if (attempt.getTaskStatus().
          equals(TaskStatus.State.SUCCEEDED.toString())) {
        long avgFinishTime = (attempt.getFinishTime() -
            attempt.getStartTime());
        if (attempt.getTaskType().equals(TaskType.MAP)) {
          mapTasks[mapIndex++] = attempt; 
          avgMapTime += avgFinishTime;
        } else if (attempt.getTaskType().equals(TaskType.REDUCE)) {
          reduceTasks[reduceIndex++] = attempt;
          avgShuffleTime += (attempt.getShuffleFinishTime() - 
              attempt.getStartTime());
          avgReduceTime += (attempt.getFinishTime() -
              attempt.getShuffleFinishTime());
        }
        break;
      }
    }
  }
  if (finishedMaps > 0) {
    avgMapTime /= finishedMaps;
  }
  if (finishedReduces > 0) {
    avgReduceTime /= finishedReduces;
    avgShuffleTime /= finishedReduces;
  }
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo; //导入方法依赖的package包/类
@Test(timeout = 30000)
public void testHistoryParsingForFailedAttempts() throws Exception {
  LOG.info("STARTING testHistoryParsingForFailedAttempts");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);
    RackResolver.init(conf);
    MRApp app = new MRAppWithHistoryWithFailedAttempt(2, 1, true, this
        .getClass().getName(), true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);

    String jobhistoryDir = JobHistoryUtils
        .getHistoryIntermediateDoneDirForUser(conf);
    JobHistory jobHistory = new JobHistory();
    jobHistory.init(conf);

    JobIndexInfo jobIndexInfo = jobHistory.getJobFileInfo(jobId)
        .getJobIndexInfo();
    String jobhistoryFileName = FileNameIndexUtils
        .getDoneFileName(jobIndexInfo);

    Path historyFilePath = new Path(jobhistoryDir, jobhistoryFileName);
    FSDataInputStream in = null;
    FileContext fc = null;
    try {
      fc = FileContext.getFileContext(conf);
      in = fc.open(fc.makeQualified(historyFilePath));
    } catch (IOException ioe) {
      LOG.info("Can not open history file: " + historyFilePath, ioe);
      throw (new Exception("Can not open History File"));
    }

    JobHistoryParser parser = new JobHistoryParser(in);
    JobInfo jobInfo = parser.parse();
    Exception parseException = parser.getParseException();
    Assert.assertNull("Caught an expected exception " + parseException,
        parseException);
    int noOffailedAttempts = 0;
    Map<TaskID, TaskInfo> allTasks = jobInfo.getAllTasks();
    for (Task task : job.getTasks().values()) {
      TaskInfo taskInfo = allTasks.get(TypeConverter.fromYarn(task.getID()));
      for (TaskAttempt taskAttempt : task.getAttempts().values()) {
        TaskAttemptInfo taskAttemptInfo = taskInfo.getAllTaskAttempts().get(
            TypeConverter.fromYarn((taskAttempt.getID())));
        // Verify rack-name for all task attempts
        Assert.assertEquals("rack-name is incorrect",
            taskAttemptInfo.getRackname(), RACK_NAME);
        if (taskAttemptInfo.getTaskStatus().equals("FAILED")) {
          noOffailedAttempts++;
        }
      }
    }
    Assert.assertEquals("No of Failed tasks doesn't match.", 2,
        noOffailedAttempts);
  } finally {
    LOG.info("FINISHED testHistoryParsingForFailedAttempts");
  }
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo; //导入方法依赖的package包/类
/** Create summary information for the parsed job */
public SummarizedJob(JobInfo job) {
  tasks = job.getAllTasks();

  for (JobHistoryParser.TaskInfo task : tasks.values()) {
    Map<TaskAttemptID, JobHistoryParser.TaskAttemptInfo> attempts = 
      task.getAllTaskAttempts();
    //allHosts.put(task.getHo(Keys.RPC_ADDRESSES), "");
    for (JobHistoryParser.TaskAttemptInfo attempt : attempts.values()) {
      long startTime = attempt.getStartTime(); 
      long finishTime = attempt.getFinishTime();
      if (attempt.getTaskType().equals(TaskType.MAP)) {
        if (mapStarted== 0 || mapStarted > startTime) {
          mapStarted = startTime; 
        }
        if (mapFinished < finishTime) {
          mapFinished = finishTime; 
        }
        totalMaps++; 
        if (attempt.getTaskStatus().equals
            (TaskStatus.State.FAILED.toString())) {
          numFailedMaps++; 
        } else if (attempt.getTaskStatus().equals
            (TaskStatus.State.KILLED.toString())) {
          numKilledMaps++;
        }
      } else if (attempt.getTaskType().equals(TaskType.REDUCE)) {
        if (reduceStarted==0||reduceStarted > startTime) {
          reduceStarted = startTime; 
        }
        if (reduceFinished < finishTime) {
          reduceFinished = finishTime; 
        }
        totalReduces++; 
        if (attempt.getTaskStatus().equals
            (TaskStatus.State.FAILED.toString())) {
          numFailedReduces++; 
        } else if (attempt.getTaskStatus().equals
            (TaskStatus.State.KILLED.toString())) {
          numKilledReduces++;
        }
      } else if (attempt.getTaskType().equals(TaskType.JOB_CLEANUP)) {
        if (cleanupStarted==0||cleanupStarted > startTime) {
          cleanupStarted = startTime; 
        }
        if (cleanupFinished < finishTime) {
          cleanupFinished = finishTime; 
        }
        totalCleanups++; 
        if (attempt.getTaskStatus().equals
            (TaskStatus.State.SUCCEEDED.toString())) {
          numFinishedCleanups++; 
        } else if (attempt.getTaskStatus().equals
            (TaskStatus.State.FAILED.toString())) {
          numFailedCleanups++;
        } else if (attempt.getTaskStatus().equals
            (TaskStatus.State.KILLED.toString())) {
          numKilledCleanups++;
        }
      } else if (attempt.getTaskType().equals(TaskType.JOB_SETUP)) {
        if (setupStarted==0||setupStarted > startTime) {
          setupStarted = startTime; 
        }
        if (setupFinished < finishTime) {
          setupFinished = finishTime; 
        }
        totalSetups++; 
        if (attempt.getTaskStatus().equals
            (TaskStatus.State.SUCCEEDED.toString())) {
          numFinishedSetups++;
        } else if (attempt.getTaskStatus().equals
            (TaskStatus.State.FAILED.toString())) {
          numFailedSetups++;
        } else if (attempt.getTaskStatus().equals
            (TaskStatus.State.KILLED.toString())) {
          numKilledSetups++;
        }
      }
    }
  }
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo; //导入方法依赖的package包/类
private static void validateTaskLevelKeyValuesFormat(JobInfo job,
                                boolean splitsCanBeEmpty) {
  Map<TaskID, TaskInfo> tasks = job.getAllTasks();

  // validate info of each task
  for (TaskInfo task : tasks.values()) {

    TaskID tid = task.getTaskId();
    long startTime = task.getStartTime();
    assertTrue("Invalid Start time", startTime > 0);
    
    long finishTime = task.getFinishTime();
    assertTrue("Task FINISH_TIME is < START_TIME in history file",
               startTime < finishTime);

    // Make sure that the Task type exists and it is valid
    TaskType type = task.getTaskType();
    assertTrue("Unknown Task type \"" + type + "\" is seen in " +
               "history file for task " + tid,
               (type.equals(TaskType.MAP) || 
                type.equals(TaskType.REDUCE) ||
                type.equals(TaskType.JOB_CLEANUP) || 
                type.equals(TaskType.JOB_SETUP)));

    if (type.equals(TaskType.MAP)) {
      String splits = task.getSplitLocations();
      //order in the condition OR check is important here
      if (!splitsCanBeEmpty || splits.length() != 0) {
        Matcher m = splitsPattern.matcher(splits);
        assertTrue("Unexpected format of SPLITS \"" + splits + "\" is seen" +
                   " in history file for task " + tid, m.matches());
      }
    }

    // Validate task status
    String status = task.getTaskStatus();
    assertTrue("Unexpected TASK_STATUS \"" + status + "\" is seen in" +
               " history file for task " + tid, (status.equals("SUCCEEDED") ||
               status.equals("FAILED") || status.equals("KILLED")));
  }
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo; //导入方法依赖的package包/类
private static void validateTaskAttemptLevelKeyValuesFormat(JobInfo job) {
  Map<TaskID, TaskInfo> tasks = job.getAllTasks();

  // For each task
  for (TaskInfo task : tasks.values()) {
    // validate info of each attempt
    for (TaskAttemptInfo attempt : task.getAllTaskAttempts().values()) {

      TaskAttemptID id = attempt.getAttemptId();
      assertNotNull(id);
      
      long startTime = attempt.getStartTime();
      assertTrue("Invalid Start time", startTime > 0);

      long finishTime = attempt.getFinishTime();
      assertTrue("Task FINISH_TIME is < START_TIME in history file",
          startTime < finishTime);

      // Make sure that the Task type exists and it is valid
      TaskType type = attempt.getTaskType();
      assertTrue("Unknown Task type \"" + type + "\" is seen in " +
                 "history file for task attempt " + id,
                 (type.equals(TaskType.MAP) || type.equals(TaskType.REDUCE) ||
                  type.equals(TaskType.JOB_CLEANUP) || 
                  type.equals(TaskType.JOB_SETUP)));

      // Validate task status
      String status = attempt.getTaskStatus();
      assertTrue("Unexpected TASK_STATUS \"" + status + "\" is seen in" +
                 " history file for task attempt " + id,
                 (status.equals(TaskStatus.State.SUCCEEDED.toString()) ||
                  status.equals(TaskStatus.State.FAILED.toString()) ||
                  status.equals(TaskStatus.State.KILLED.toString())));

      // Successful Reduce Task Attempts should have valid SHUFFLE_FINISHED
      // time and SORT_FINISHED time
      if (type.equals(TaskType.REDUCE) && 
          status.equals(TaskStatus.State.SUCCEEDED.toString())) {
        long shuffleFinishTime = attempt.getShuffleFinishTime();
        assertTrue(startTime < shuffleFinishTime);
        
        long sortFinishTime = attempt.getSortFinishTime();
        assertTrue(shuffleFinishTime < sortFinishTime);
      }
      else if (type.equals(TaskType.MAP) && 
          status.equals(TaskStatus.State.SUCCEEDED.toString())) {
        // Successful MAP Task Attempts should have valid MAP_FINISHED time
       long mapFinishTime = attempt.getMapFinishTime();
       assertTrue(startTime < mapFinishTime);
      }

      // check if hostname is valid
      String hostname = attempt.getHostname();
      Matcher m = hostNamePattern.matcher(hostname);
      assertTrue("Unexpected Host name of task attempt " + id, m.matches());

      // check if trackername is valid
      String trackerName = attempt.getTrackerName();
      m = trackerNamePattern.matcher(trackerName);
      assertTrue("Unexpected tracker name of task attempt " + id,
                 m.matches());

      if (!status.equals("KILLED")) {
        // check if http port is valid
        int httpPort = attempt.getHttpPort();
        assertTrue(httpPort > 0);
      }
      
      // check if counters are parsable
      Counters counters = attempt.getCounters();
      assertNotNull(counters);
    }
  }
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo; //导入方法依赖的package包/类
private static void validateTaskLevelKeyValues(MiniMRCluster mr,
    RunningJob job, JobInfo jobInfo) throws IOException  {

  JobTracker jt = mr.getJobTrackerRunner().getJobTracker();
  JobInProgress jip = jt.getJob(job.getID());

  // Get the 1st map, 1st reduce, cleanup & setup taskIDs and
  // validate their history info
  TaskID mapTaskId = new TaskID(job.getID(), TaskType.MAP, 0);
  TaskID reduceTaskId = new TaskID(job.getID(), TaskType.REDUCE, 0);

  TaskInProgress cleanups[] = jip.cleanup;
  TaskID cleanupTaskId;
  if (cleanups[0].isComplete()) {
    cleanupTaskId = cleanups[0].getTIPId();
  }
  else {
    cleanupTaskId = cleanups[1].getTIPId();
  }

  TaskInProgress setups[] = jip.setup;
  TaskID setupTaskId;
  if (setups[0].isComplete()) {
    setupTaskId = setups[0].getTIPId();
  }
  else {
    setupTaskId = setups[1].getTIPId();
  }

  Map<TaskID, TaskInfo> tasks = jobInfo.getAllTasks();

  // validate info of the 4 tasks(cleanup, setup, 1st map, 1st reduce)    

  for (TaskInfo task : tasks.values()) {
    TaskID tid = task.getTaskId();

    if (tid.equals(mapTaskId) ||
        tid.equals(reduceTaskId) ||
        tid.equals(cleanupTaskId) ||
        tid.equals(setupTaskId)) {

      TaskInProgress tip = jip.getTaskInProgress
      (org.apache.hadoop.mapred.TaskID.downgrade(tid));
      assertTrue("START_TIME of Task " + tid + " obtained from history " +
          "file did not match the expected value", 
          tip.getExecStartTime() ==
            task.getStartTime());

      assertTrue("FINISH_TIME of Task " + tid + " obtained from history " +
          "file did not match the expected value",
          tip.getExecFinishTime() ==
            task.getFinishTime());

      if (tid == mapTaskId) {//check splits only for map task
        assertTrue("Splits of Task " + tid + " obtained from history file " +
            " did not match the expected value",
            tip.getSplitNodes().equals(task.getSplitLocations()));
      }

      TaskAttemptID attemptId = tip.getSuccessfulTaskid();
      TaskStatus ts = tip.getTaskStatus(
          org.apache.hadoop.mapred.TaskAttemptID.downgrade(attemptId));

      // Validate task counters
      Counters c = new Counters(ts.getCounters());
      assertTrue("Counters of Task " + tid + " obtained from history file " +
          " did not match the expected value",
          c.equals(task.getCounters()));
    }
  }
}