Java JobHistoryParser类代码示例

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入依赖的package包/类
@Test
public void testFailedJobHistoryWithoutDiagnostics() throws Exception {
  final Path histPath = new Path(getClass().getClassLoader().getResource(
      "job_1393307629410_0001-1393307687476-user-Sleep+job-1393307723835-0-0-FAILED-default-1393307693920.jhist")
      .getFile());
  final FileSystem lfs = FileSystem.getLocal(new Configuration());
  final FSDataInputStream fsdis = lfs.open(histPath);
  try {
    JobHistoryParser parser = new JobHistoryParser(fsdis);
    JobInfo info = parser.parse();
    assertEquals("History parsed jobId incorrectly",
        info.getJobId(), JobID.forName("job_1393307629410_0001") );
    assertEquals("Default diagnostics incorrect ", "", info.getErrorInfo());
  } finally {
    fsdis.close();
  }
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入依赖的package包/类
private long[] getTaskExecTime(JobHistoryParser.TaskAttemptInfo attempInfo) {
  long startTime = attempInfo.getStartTime();
  long finishTime = attempInfo.getFinishTime();
  boolean isMapper = (attempInfo.getTaskType() == TaskType.MAP);

  long[] time;
  if (isMapper) {
    time = new long[]{finishTime - startTime, 0, 0, startTime, finishTime};
  } else {
    long shuffleFinishTime = attempInfo.getShuffleFinishTime();
    long mergeFinishTime = attempInfo.getSortFinishTime();
    time = new long[]{finishTime - startTime, shuffleFinishTime - startTime,
            mergeFinishTime - shuffleFinishTime, startTime, finishTime};
  }
  return time;
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入依赖的package包/类
@Test (timeout=30000)
public void testCompletedJobWithDiagnostics() throws Exception {
  final String jobError = "Job Diagnostics";
  JobInfo jobInfo = spy(new JobInfo());
  when(jobInfo.getErrorInfo()).thenReturn(jobError);
  when(jobInfo.getJobStatus()).thenReturn(JobState.FAILED.toString());
  when(jobInfo.getAMInfos()).thenReturn(Collections.<JobHistoryParser.AMInfo>emptyList());
  final JobHistoryParser mockParser = mock(JobHistoryParser.class);
  when(mockParser.parse()).thenReturn(jobInfo);
  HistoryFileInfo info = mock(HistoryFileInfo.class);
  when(info.getConfFile()).thenReturn(fullConfPath);
  when(info.getHistoryFile()).thenReturn(fullHistoryPath);
  CompletedJob job =
    new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user",
        info, jobAclsManager) {
          @Override
          protected JobHistoryParser createJobHistoryParser(
              Path historyFileAbsolute) throws IOException {
             return mockParser;
          }
  };
  assertEquals(jobError, job.getReport().getDiagnostics());
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入依赖的package包/类
/**
 * Get the simulated job history of a job.
 * @param simulatedJobID - simulated job id.
 * @return - simulated job information.
 * @throws IOException - if an I/O error occurs.
 */
public JobHistoryParser.JobInfo getSimulatedJobHistory(JobID simulatedJobID) 
    throws IOException {
  FileSystem fs = null;
  try {
    String historyFilePath = jtClient.getProxy().
        getJobHistoryLocationForRetiredJob(simulatedJobID);
    Path jhpath = new Path(historyFilePath);
    fs = jhpath.getFileSystem(conf);
    JobHistoryParser jhparser = new JobHistoryParser(fs, jhpath);
    JobHistoryParser.JobInfo jhInfo = jhparser.parse();
    return jhInfo;

  } finally {
    fs.close();
  }
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入依赖的package包/类
@Override
public List<AMInfo> getAMInfos() {
  List<AMInfo> amInfos = new LinkedList<AMInfo>();
  for (org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.AMInfo jhAmInfo : jobInfo
      .getAMInfos()) {
    AMInfo amInfo =
        MRBuilderUtils.newAMInfo(jhAmInfo.getAppAttemptId(),
            jhAmInfo.getStartTime(), jhAmInfo.getContainerId(),
            jhAmInfo.getNodeManagerHost(), jhAmInfo.getNodeManagerPort(),
            jhAmInfo.getNodeManagerHttpPort());
 
    amInfos.add(amInfo);
  }
  return amInfos;
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入依赖的package包/类
/**
 * Test compatibility of JobHistoryParser with 2.0.3-alpha history files
 * @throws IOException
 */
@Test
public void testTaskAttemptUnsuccessfulCompletionWithoutCounters203() throws IOException 
  { 
    Path histPath = new Path(getClass().getClassLoader().getResource(
      "job_2.0.3-alpha-FAILED.jhist").getFile());
    JobHistoryParser parser = new JobHistoryParser(FileSystem.getLocal
        (new Configuration()), histPath);
    JobInfo jobInfo = parser.parse(); 
    LOG.info(" job info: " + jobInfo.getJobname() + " "
      + jobInfo.getFinishedMaps() + " " 
      + jobInfo.getTotalMaps() + " " 
      + jobInfo.getJobId() ) ;
  }
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入依赖的package包/类
/**
 * Test compatibility of JobHistoryParser with 2.4.0 history files
 * @throws IOException
 */
@Test
public void testTaskAttemptUnsuccessfulCompletionWithoutCounters240() throws IOException 
  {
    Path histPath = new Path(getClass().getClassLoader().getResource(
      "job_2.4.0-FAILED.jhist").getFile());
    JobHistoryParser parser = new JobHistoryParser(FileSystem.getLocal
        (new Configuration()), histPath);
    JobInfo jobInfo = parser.parse(); 
    LOG.info(" job info: " + jobInfo.getJobname() + " "
      + jobInfo.getFinishedMaps() + " "
      + jobInfo.getTotalMaps() + " "
      + jobInfo.getJobId() );
  }
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入依赖的package包/类
/**
 * Test compatibility of JobHistoryParser with 0.23.9 history files
 * @throws IOException
 */
@Test
public void testTaskAttemptUnsuccessfulCompletionWithoutCounters0239() throws IOException 
  {
    Path histPath = new Path(getClass().getClassLoader().getResource(
        "job_0.23.9-FAILED.jhist").getFile());
    JobHistoryParser parser = new JobHistoryParser(FileSystem.getLocal
        (new Configuration()), histPath);
    JobInfo jobInfo = parser.parse(); 
    LOG.info(" job info: " + jobInfo.getJobname() + " "
      + jobInfo.getFinishedMaps() + " " 
      + jobInfo.getTotalMaps() + " " 
      + jobInfo.getJobId() ) ;
    }
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入依赖的package包/类
protected MapReduceTaskData[] getTaskData(String jobId, List<JobHistoryParser.TaskInfo> infoList) {
  int sampleSize = sampleAndGetSize(jobId, infoList);

  List<MapReduceTaskData> taskList = new ArrayList<MapReduceTaskData>();
  for (int i = 0; i < sampleSize; i++) {
    JobHistoryParser.TaskInfo tInfo = infoList.get(i);

    String taskId = tInfo.getTaskId().toString();
    TaskAttemptID attemptId = null;
    if(tInfo.getTaskStatus().equals("SUCCEEDED")) {
      attemptId = tInfo.getSuccessfulAttemptId();
    } else {
      attemptId = tInfo.getFailedDueToAttemptId();
    }

    MapReduceTaskData taskData = new MapReduceTaskData(taskId, attemptId == null ? "" : attemptId.toString() , tInfo.getTaskStatus());

    MapReduceCounterData taskCounterData = getCounterData(tInfo.getCounters());

    long[] taskExecTime = null;
    if (attemptId != null) {
      taskExecTime = getTaskExecTime(tInfo.getAllTaskAttempts().get(attemptId));
    }

    taskData.setTimeAndCounter(taskExecTime, taskCounterData);
    taskList.add(taskData);
  }
  return taskList.toArray(new MapReduceTaskData[taskList.size()]);
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入依赖的package包/类
@Test
public void testGetTaskData() {
  FetcherConfiguration fetcherConf = new FetcherConfiguration(document9.getDocumentElement());

  try {
    MapReduceFSFetcherHadoop2 fetcher = new MapReduceFSFetcherHadoop2(
            fetcherConf.getFetchersConfigurationData().get(0));
    String jobId = "job_14000_001";
    List<JobHistoryParser.TaskInfo> infoList = new ArrayList<JobHistoryParser.TaskInfo>();
    infoList.add(new MockTaskInfo(1, true));
    infoList.add(new MockTaskInfo(2, false));

    MapReduceTaskData[] taskList = fetcher.getTaskData(jobId, infoList);
    Assert.assertNotNull("taskList should not be null.", taskList);
    int succeededTaskCount = 0;
    for (MapReduceTaskData task : taskList) {
      Assert.assertNotNull("Null pointer in taskList.", task);
      if(task.getState().equals("SUCCEEDED")) {
        succeededTaskCount++;
      }
    }
    Assert.assertEquals("Should have total two tasks.", 2, taskList.length);
    Assert.assertEquals("Should have only one succeeded task.", 1, succeededTaskCount);
  } catch (IOException e) {
    Assert.assertNull("Failed to initialize FileSystem.", e);
  }
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入依赖的package包/类
public MockTaskInfo(int id, boolean succeeded) {
  this.taskId = new TaskID("job1", 1, TaskType.MAP, id);
  this.taskType = TaskType.MAP;
  this.succeeded = succeeded;
  this.counters = new Counters();
  this.finishTime = System.currentTimeMillis();
  this.startTime = finishTime - 10000;
  this.failedDueToAttemptId = new TaskAttemptID(taskId, 0);
  this.successfulAttemptId = new TaskAttemptID(taskId, 1);
  this.attemptsMap = new HashMap<TaskAttemptID, JobHistoryParser.TaskAttemptInfo>();
  this.attemptsMap.put(failedDueToAttemptId, new JobHistoryParser.TaskAttemptInfo());
  this.attemptsMap.put(successfulAttemptId, new JobHistoryParser.TaskAttemptInfo());
}
 

import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser; //导入依赖的package包/类
protected synchronized void loadFullHistoryData(boolean loadTasks,
    Path historyFileAbsolute) throws IOException {
  LOG.info("Loading history file: [" + historyFileAbsolute + "]");
  if (this.jobInfo != null) {
    return;
  }
  
  if (historyFileAbsolute != null) {
    JobHistoryParser parser = null;
    try {
      parser = createJobHistoryParser(historyFileAbsolute);
      this.jobInfo = parser.parse();
    } catch (IOException e) {
      throw new YarnRuntimeException("Could not load history file "
          + historyFileAbsolute, e);
    }
    IOException parseException = parser.getParseException(); 
    if (parseException != null) {
      throw new YarnRuntimeException(
          "Could not parse history file " + historyFileAbsolute, 
          parseException);
    }
  } else {
    throw new IOException("History file not found");
  }
  if (loadTasks) {
    loadAllTasks();
    LOG.info("TaskInfo loaded");
  }    
}