Java JobHistoryParser.JobInfo方法代码示例

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入方法依赖的package包/类
/**
 * Get the simulated job history of a job.
 * @param simulatedJobID - simulated job id.
 * @return - simulated job information.
 * @throws IOException - if an I/O error occurs.
 */
public JobHistoryParser.JobInfo getSimulatedJobHistory(JobID simulatedJobID) 
    throws IOException {
  FileSystem fs = null;
  try {
    String historyFilePath = jtClient.getProxy().
        getJobHistoryLocationForRetiredJob(simulatedJobID);
    Path jhpath = new Path(historyFilePath);
    fs = jhpath.getFileSystem(conf);
    JobHistoryParser jhparser = new JobHistoryParser(fs, jhpath);
    JobHistoryParser.JobInfo jhInfo = jhparser.parse();
    return jhInfo;

  } finally {
    fs.close();
  }
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入方法依赖的package包/类
/**
 * Checks if the history file has expected job status
 * @param id job id
 * @param conf job conf
 */
private static void validateJobHistoryJobStatus(JobHistory jobHistory,
    JobID id, JobConf conf, String status) throws IOException  {

  // Get the history file name
  Path doneDir = jobHistory.getCompletedJobHistoryLocation();
  String logFileName = getDoneFile(jobHistory, conf, id, doneDir);
  
  // Framework history log file location
  Path logFile = new Path(doneDir, logFileName);
  FileSystem fileSys = logFile.getFileSystem(conf);
 
  // Check if the history file exists
  assertTrue("History file does not exist", fileSys.exists(logFile));

  // check history file permission
  assertTrue("History file permissions does not match", 
  fileSys.getFileStatus(logFile).getPermission().equals(
     new FsPermission(JobHistory.HISTORY_FILE_PERMISSION)));
  
  JobHistoryParser parser = new JobHistoryParser(fileSys, 
      logFile.toUri().getPath());
  JobHistoryParser.JobInfo jobInfo = parser.parse();
  

  assertTrue("Job Status read from job history file is not the expected" +
       " status", status.equals(jobInfo.getJobStatus()));
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入方法依赖的package包/类
/**
 * It verifies the gridmix simulated job summary.
 * @param zombieJob - Original job summary.
 * @param jhInfo  - Simulated job history info.
 * @param jobConf - simulated job configuration.
 * @throws IOException - if an I/O error occurs.
 */
public void verifySimulatedJobSummary(ZombieJob zombieJob, 
   JobHistoryParser.JobInfo jhInfo, JobConf jobConf) throws IOException {
  Assert.assertEquals("Job id has not matched", zombieJob.getJobID(), 
                      JobID.forName(jobConf.get(origJobIdKey)));

  Assert.assertEquals("Job maps have not matched", zombieJob.getNumberMaps(),
                      jhInfo.getTotalMaps());

  if (!jobConf.getBoolean(mapTaskKey, false)) { 
    Assert.assertEquals("Job reducers have not matched", 
        zombieJob.getNumberReduces(), jhInfo.getTotalReduces());
  } else {
    Assert.assertEquals("Job reducers have not matched",
                        0, jhInfo.getTotalReduces());
  }

  Assert.assertEquals("Job status has not matched.", 
                      zombieJob.getOutcome().name(), 
                      convertJobStatus(jhInfo.getJobStatus()));

  LoggedJob loggedJob = zombieJob.getLoggedJob();
  Assert.assertEquals("Job priority has not matched.", 
                      loggedJob.getPriority().toString(), 
                      jhInfo.getPriority());

  if (jobConf.get(usrResolver).contains("RoundRobin")) {
     String user = UserGroupInformation.getLoginUser().getShortUserName();
     Assert.assertTrue(jhInfo.getJobId().toString() 
                      + " has not impersonate with other user.", 
                      !jhInfo.getUsername().equals(user));
  }
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入方法依赖的package包/类
@Override
public MapReduceApplicationData fetchData(AnalyticJob job) throws IOException {
  DataFiles files = getHistoryFiles(job);
  String confFile = files.getJobConfPath();
  String histFile = files.getJobHistPath();
  String appId = job.getAppId();
  String jobId = Utils.getJobIdFromApplicationId(appId);

  MapReduceApplicationData jobData = new MapReduceApplicationData();
  jobData.setAppId(appId).setJobId(jobId);

  // Fetch job config
  Configuration jobConf = new Configuration(false);
  jobConf.addResource(_fs.open(new Path(confFile)), confFile);
  Properties jobConfProperties = new Properties();
  for (Map.Entry<String, String> entry : jobConf) {
    jobConfProperties.put(entry.getKey(), entry.getValue());
  }
  jobData.setJobConf(jobConfProperties);

  // Check if job history file is too large and should be throttled
  if (_fs.getFileStatus(new Path(histFile)).getLen() > _maxLogSizeInMB * FileUtils.ONE_MB) {
    String errMsg = "The history log of MapReduce application: " + appId + " is over the limit size of "
            + _maxLogSizeInMB + " MB, the parsing process gets throttled.";
    logger.warn(errMsg);
    jobData.setDiagnosticInfo(errMsg);
    jobData.setSucceeded(false);  // set succeeded to false to avoid heuristic analysis
    return jobData;
  }

  // Analyze job history file
  JobHistoryParser parser = new JobHistoryParser(_fs, histFile);
  JobHistoryParser.JobInfo jobInfo = parser.parse();
  IOException parseException = parser.getParseException();
  if (parseException != null) {
    throw new RuntimeException("Could not parse history file " + histFile, parseException);
  }

  jobData.setSubmitTime(jobInfo.getSubmitTime());
  jobData.setStartTime(jobInfo.getLaunchTime());
  jobData.setFinishTime(jobInfo.getFinishTime());

  String state = jobInfo.getJobStatus();
  if (state.equals("SUCCEEDED")) {
    jobData.setSucceeded(true);
  }
  else if (state.equals("FAILED")) {
    jobData.setSucceeded(false);
    jobData.setDiagnosticInfo(jobInfo.getErrorInfo());
  } else {
    throw new RuntimeException("job neither succeeded or failed. can not process it ");
  }


  // Fetch job counter
  MapReduceCounterData jobCounter = getCounterData(jobInfo.getTotalCounters());

  // Fetch task data
  Map<TaskID, JobHistoryParser.TaskInfo> allTasks = jobInfo.getAllTasks();
  List<JobHistoryParser.TaskInfo> mapperInfoList = new ArrayList<JobHistoryParser.TaskInfo>();
  List<JobHistoryParser.TaskInfo> reducerInfoList = new ArrayList<JobHistoryParser.TaskInfo>();
  for (JobHistoryParser.TaskInfo taskInfo : allTasks.values()) {
    if (taskInfo.getTaskType() == TaskType.MAP) {
      mapperInfoList.add(taskInfo);
    } else {
      reducerInfoList.add(taskInfo);
    }
  }
  if (jobInfo.getTotalMaps() > MAX_SAMPLE_SIZE) {
    logger.debug(jobId + " total mappers: " + mapperInfoList.size());
  }
  if (jobInfo.getTotalReduces() > MAX_SAMPLE_SIZE) {
    logger.debug(jobId + " total reducers: " + reducerInfoList.size());
  }
  MapReduceTaskData[] mapperList = getTaskData(jobId, mapperInfoList);
  MapReduceTaskData[] reducerList = getTaskData(jobId, reducerInfoList);

  jobData.setCounters(jobCounter).setMapperData(mapperList).setReducerData(reducerList);

  return jobData;
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入方法依赖的package包/类
/**
 *  Validates the format of contents of history file
 *  (1) history file exists and in correct location
 *  (2) Verify if the history file is parsable
 *  (3) Validate the contents of history file
 *     (a) Format of all TIMEs are checked against a regex
 *     (b) validate legality/format of job level key, values
 *     (c) validate legality/format of task level key, values
 *     (d) validate legality/format of attempt level key, values
 *     (e) check if all the TaskAttempts, Tasks started are finished.
 *         Check finish of each TaskAttemptID against its start to make sure
 *         that all TaskAttempts, Tasks started are indeed finished and the
 *         history log lines are in the proper order.
 *         We want to catch ordering of history lines like
 *            Task START
 *            Attempt START
 *            Task FINISH
 *            Attempt FINISH
 *         (speculative execution is turned off for this).
 * @param id job id
 * @param conf job conf
 */
public static void validateJobHistoryFileFormat(JobHistory jobHistory,
    JobID id, JobConf conf,
               String status, boolean splitsCanBeEmpty) throws IOException  {

  // Get the history file name
  Path dir = jobHistory.getCompletedJobHistoryLocation();
  String logFileName = getDoneFile(jobHistory, conf, id, dir);

  // Framework history log file location
  Path logFile = new Path(dir, logFileName);
  FileSystem fileSys = logFile.getFileSystem(conf);
 
  // Check if the history file exists
  assertTrue("History file does not exist", fileSys.exists(logFile));

  JobHistoryParser parser = new JobHistoryParser(fileSys, 
      logFile.toUri().getPath());
  JobHistoryParser.JobInfo jobInfo = parser.parse();

  // validate format of job level key, values
  validateJobLevelKeyValuesFormat(jobInfo, status);

  // validate format of task level key, values
  validateTaskLevelKeyValuesFormat(jobInfo, splitsCanBeEmpty);

  // validate format of attempt level key, values
  validateTaskAttemptLevelKeyValuesFormat(jobInfo);

  // check if all the TaskAttempts, Tasks started are finished for
  // successful jobs
  if (status.equals("SUCCEEDED")) {
    // Make sure that the lists in taskIDsToAttemptIDs are empty.
    for(Iterator<String> it = 
      taskIDsToAttemptIDs.keySet().iterator();it.hasNext();) {
      String taskid = it.next();
      assertTrue("There are some Tasks which are not finished in history " +
                 "file.", taskEnds.contains(taskid));
      List<String> attemptIDs = taskIDsToAttemptIDs.get(taskid);
      if(attemptIDs != null) {
        assertTrue("Unexpected. TaskID " + taskid + " has task attempt(s)" +
                   " that are not finished.", (attemptIDs.size() == 1));
      }
    }
  }
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入方法依赖的package包/类
/**
 * Checks if the history file content is as expected comparing with the
 * actual values obtained from JT.
 * Job Level, Task Level and Task Attempt Level Keys, Values are validated.
 * @param job RunningJob object of the job whose history is to be validated
 * @param conf job conf
 */
public static void validateJobHistoryFileContent(MiniMRCluster mr,
                            RunningJob job, JobConf conf) throws IOException  {

  JobID id = job.getID();
  JobHistory jobHistory = 
    mr.getJobTrackerRunner().getJobTracker().getJobHistory();
  Path doneDir = jobHistory.getCompletedJobHistoryLocation();
  // Get the history file name
  String logFileName = getDoneFile(jobHistory, conf, id, doneDir);

  // Framework history log file location
  Path logFile = new Path(doneDir, logFileName);
  FileSystem fileSys = logFile.getFileSystem(conf);
 
  // Check if the history file exists
  assertTrue("History file does not exist", fileSys.exists(logFile));

  JobHistoryParser parser = new JobHistoryParser(fileSys,
      logFile.toUri().getPath());
  
  JobHistoryParser.JobInfo jobInfo = parser.parse();
  // Now the history file contents are available in jobInfo. Let us compare
  // them with the actual values from JT.
  validateJobLevelKeyValues(mr, job, jobInfo, conf);
  validateTaskLevelKeyValues(mr, job, jobInfo);
  validateTaskAttemptLevelKeyValues(mr, job, jobInfo);
  
  // Also JobACLs should be correct
  if (mr.getJobTrackerRunner().getJobTracker()
      .areACLsEnabled()) {
    AccessControlList acl = new AccessControlList(
        conf.get(JobACL.VIEW_JOB.getAclName(), " "));
    assertTrue("VIEW_JOB ACL is not properly logged to history file.",
        acl.toString().equals(
        jobInfo.getJobACLs().get(JobACL.VIEW_JOB).toString()));
    acl = new AccessControlList(
        conf.get(JobACL.MODIFY_JOB.getAclName(), " "));
    assertTrue("MODIFY_JOB ACL is not properly logged to history file.",
        acl.toString().equals(
        jobInfo.getJobACLs().get(JobACL.MODIFY_JOB).toString()));
  }
  
  // Validate the job queue name
  assertTrue(jobInfo.getJobQueueName().equals(conf.getQueueName()));
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入方法依赖的package包/类
private JobID validateJobRetire(JobConf jobConf, Path inDir, Path outDir, 
    JobTracker jobtracker) throws IOException {

  RunningJob rj = UtilsForTests.runJob(jobConf, inDir, outDir, 0, 0);
  rj.waitForCompletion();
  assertTrue(rj.isSuccessful());
  JobID id = rj.getID();

  //wait for job to get retired
  waitTillRetire(id, jobtracker);
  
  assertTrue("History url not set", rj.getHistoryUrl() != null && 
  rj.getHistoryUrl().length() > 0);
  assertNotNull("Job is not in cache", jobtracker.getJobStatus(id));
  
  // get the job conf filename
  String name = jobtracker.getLocalJobFilePath(id);
  File file = new File(name);
 
  assertFalse("JobConf file not deleted", file.exists());
  
  // test redirections
  final String JOBDETAILS = "jobdetails";
  final String JOBCONF = "jobconf";
  final String JOBTASKS = "jobtasks";
  final String TASKSTATS = "taskstats";
  final String TASKDETAILS = "taskdetails";

  // test redirections of job related pages
  String jobUrlStr = rj.getTrackingURL();
  URL jobUrl = new URL(jobUrlStr);
  URL jobConfUrl = new URL(jobUrlStr.replace(JOBDETAILS, JOBCONF));
  URL jobTasksUrl = new URL(jobUrlStr.replace(JOBDETAILS, JOBTASKS)
                            + "&type=map&pagenum=1");
  verifyRedirection(jobConfUrl);
  verifyRedirection(jobTasksUrl);
  verifyRedirection(jobUrl);

  // test redirections of task and task attempt pages
  String jobTrackerUrlStr =
      jobUrlStr.substring(0, jobUrlStr.indexOf(JOBDETAILS));
  Path logFile = new Path(jobtracker.getJobHistory().getHistoryFilePath(id));
  JobHistoryParser.JobInfo jobInfo =
    JSPUtil.getJobInfo(logFile, logFile.getFileSystem(jobConf), jobtracker);
  for (TaskID tid : jobInfo.getAllTasks().keySet()) {
    URL taskDetailsUrl = new URL(jobTrackerUrlStr + TASKDETAILS +
                                 ".jsp?tipid=" + tid);
    // test redirections of all tasks
    verifyRedirection(taskDetailsUrl);
  }
  for (JobHistoryParser.TaskInfo task : jobInfo.getAllTasks().values()) {
    for(org.apache.hadoop.mapreduce.TaskAttemptID attemptid :
        task.getAllTaskAttempts().keySet()) {
      URL taskstats = new URL(jobTrackerUrlStr + TASKSTATS +
          ".jsp?attemptid=" + attemptid);
      // test redirections of all task attempts
      verifyRedirection(taskstats);
    }
  }
  return id;
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入方法依赖的package包/类
/**
 * It verifies the Gridmix jobs with corresponding job story in a trace file.
 * @param jobids - gridmix job ids.
 * @throws IOException - if an I/O error occurs.
 * @throws ParseException - if an parse error occurs.
 */
public void verifyGridmixJobsWithJobStories(List<JobID> jobids) 
    throws IOException, ParseException {

  SortedMap <Long, String> origSubmissionTime = new TreeMap <Long, String>();
  SortedMap <Long, String> simuSubmissionTime = new TreeMap<Long, String>();
  GridmixJobStory gjs = new GridmixJobStory(path, conf);
  final Iterator<JobID> ite = jobids.iterator();
  File destFolder = new File(System.getProperty("java.io.tmpdir") 
                            + "/gridmix-st/");
  destFolder.mkdir();
  while (ite.hasNext()) {
    JobID simuJobId = ite.next();
    JobHistoryParser.JobInfo jhInfo = getSimulatedJobHistory(simuJobId);
    Assert.assertNotNull("Job history not found.", jhInfo);
    Counters counters = jhInfo.getTotalCounters();
    JobConf simuJobConf = getSimulatedJobConf(simuJobId, destFolder);
    String origJobId = simuJobConf.get(origJobIdKey);
    LOG.info("OriginalJobID<->CurrentJobID:" 
            + origJobId + "<->" + simuJobId);

    if (userResolverVal == null) {
      userResolverVal = simuJobConf.get(usrResolver);
    }
    ZombieJob zombieJob = gjs.getZombieJob(JobID.forName(origJobId));
    Map<String, Long> mapJobCounters = getJobMapCounters(zombieJob);
    Map<String, Long> reduceJobCounters = getJobReduceCounters(zombieJob);
    if (simuJobConf.get(jobSubKey).contains("REPLAY")) {
        origSubmissionTime.put(zombieJob.getSubmissionTime(), 
                               origJobId.toString() + "^" + simuJobId); 
        simuSubmissionTime.put(jhInfo.getSubmitTime() , 
                               origJobId.toString() + "^" + simuJobId); ;
    }

    LOG.info("Verifying the job <" + simuJobId + "> and wait for a while...");
    verifySimulatedJobSummary(zombieJob, jhInfo, simuJobConf);
    verifyJobMapCounters(counters, mapJobCounters, simuJobConf);
    verifyJobReduceCounters(counters, reduceJobCounters, simuJobConf); 
    verifyCompressionEmulation(zombieJob.getJobConf(), simuJobConf, counters, 
                               reduceJobCounters, mapJobCounters);
    verifyDistributeCache(zombieJob,simuJobConf);
    setJobDistributedCacheInfo(simuJobId.toString(), simuJobConf, 
       zombieJob.getJobConf());
    LOG.info("Done.");
  }
  verifyDistributedCacheBetweenJobs(simuAndOrigJobsInfo);
}