Java JobStatus類代碼示例

import org.apache.hadoop.mapreduce.JobStatus; //導入依賴的package包/類
@Override
public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts)
		throws IOException, InterruptedException {

	addHistoryToken(ts);

	// Construct necessary information to start the MR AM
	ApplicationSubmissionContext appContext = createApplicationSubmissionContext(conf, jobSubmitDir, ts);

	// Submit to ResourceManager
	try {
		ApplicationId applicationId = resMgrDelegate.submitApplication(appContext);

		ApplicationReport appMaster = resMgrDelegate.getApplicationReport(applicationId);
		String diagnostics = (appMaster == null ? "application report is null" : appMaster.getDiagnostics());
		if (appMaster == null || appMaster.getYarnApplicationState() == YarnApplicationState.FAILED
				|| appMaster.getYarnApplicationState() == YarnApplicationState.KILLED) {
			throw new IOException("Failed to run job : " + diagnostics);
		}
		return clientCache.getClient(jobId).getJobStatus(jobId);
	} catch (YarnException e) {
		throw new IOException(e);
	}
}
 

import org.apache.hadoop.mapreduce.JobStatus; //導入依賴的package包/類
public JobStatus getJobStatus(JobID oldJobID) throws IOException {
  org.apache.hadoop.mapreduce.v2.api.records.JobId jobId =
    TypeConverter.toYarn(oldJobID);
  GetJobReportRequest request =
      recordFactory.newRecordInstance(GetJobReportRequest.class);
  request.setJobId(jobId);
  JobReport report = ((GetJobReportResponse) invoke("getJobReport",
      GetJobReportRequest.class, request)).getJobReport();
  JobStatus jobStatus = null;
  if (report != null) {
    if (StringUtils.isEmpty(report.getJobFile())) {
      String jobFile = MRApps.getJobFile(conf, report.getUser(), oldJobID);
      report.setJobFile(jobFile);
    }
    String historyTrackingUrl = report.getTrackingUrl();
    String url = StringUtils.isNotEmpty(historyTrackingUrl)
        ? historyTrackingUrl : trackingUrl;
    jobStatus = TypeConverter.fromYarn(report, url);
  }
  return jobStatus;
}
 

import org.apache.hadoop.mapreduce.JobStatus; //導入依賴的package包/類
@Test
public void testRetriesOnConnectionFailure() throws Exception {

  MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
  when(historyServerProxy.getJobReport(getJobReportRequest())).thenThrow(
      new RuntimeException("1")).thenThrow(new RuntimeException("2"))       
      .thenReturn(getJobReportResponse());

  ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
  when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
      .thenReturn(null);

  ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(
      historyServerProxy, rm);

  JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
  Assert.assertNotNull(jobStatus);
  verify(historyServerProxy, times(3)).getJobReport(
      any(GetJobReportRequest.class));
}
 

import org.apache.hadoop.mapreduce.JobStatus; //導入依賴的package包/類
@Test
public void testHistoryServerNotConfigured() throws Exception {
  //RM doesn't have app report and job History Server is not configured
  ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(
      null, getRMDelegate());
  JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
  Assert.assertEquals("N/A", jobStatus.getUsername());
  Assert.assertEquals(JobStatus.State.PREP, jobStatus.getState());

  //RM has app report and job History Server is not configured
  ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
  ApplicationReport applicationReport = getFinishedApplicationReport();
  when(rm.getApplicationReport(jobId.getAppId())).thenReturn(
      applicationReport);

  clientServiceDelegate = getClientServiceDelegate(null, rm);
  jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
  Assert.assertEquals(applicationReport.getUser(), jobStatus.getUsername());
  Assert.assertEquals(JobStatus.State.SUCCEEDED, jobStatus.getState());
}
 

import org.apache.hadoop.mapreduce.JobStatus; //導入依賴的package包/類
@Test
public void testJobReportFromHistoryServer() throws Exception {                                 
  MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);                           
  when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(                      
      getJobReportResponseFromHistoryServer());                                                 
  ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);                                     
  when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))                      
  .thenReturn(null);                                                                        
  ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(                       
      historyServerProxy, rm);

  JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
  Assert.assertNotNull(jobStatus);
  Assert.assertEquals("TestJobFilePath", jobStatus.getJobFile());                               
  Assert.assertEquals("http://TestTrackingUrl", jobStatus.getTrackingUrl());                    
  Assert.assertEquals(1.0f, jobStatus.getMapProgress(), 0.0f);
  Assert.assertEquals(1.0f, jobStatus.getReduceProgress(), 0.0f);
}
 

import org.apache.hadoop.mapreduce.JobStatus; //導入依賴的package包/類
@Test
public void tesAllJobs() throws Exception {
  final ApplicationClientProtocol applicationsManager = Mockito.mock(ApplicationClientProtocol.class);
  GetApplicationsResponse allApplicationsResponse = Records
      .newRecord(GetApplicationsResponse.class);
  List<ApplicationReport> applications = new ArrayList<ApplicationReport>();
  applications.add(getApplicationReport(YarnApplicationState.FINISHED,
      FinalApplicationStatus.FAILED));
  applications.add(getApplicationReport(YarnApplicationState.FINISHED,
      FinalApplicationStatus.SUCCEEDED));
  applications.add(getApplicationReport(YarnApplicationState.FINISHED,
      FinalApplicationStatus.KILLED));
  applications.add(getApplicationReport(YarnApplicationState.FAILED,
      FinalApplicationStatus.FAILED));
  allApplicationsResponse.setApplicationList(applications);
  Mockito.when(
      applicationsManager.getApplications(Mockito
          .any(GetApplicationsRequest.class))).thenReturn(
      allApplicationsResponse);
  ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate(
    new YarnConfiguration()) {
    @Override
    protected void serviceStart() throws Exception {
      Assert.assertTrue(this.client instanceof YarnClientImpl);
      ((YarnClientImpl) this.client).setRMClient(applicationsManager);
    }
  };
  JobStatus[] allJobs = resourceMgrDelegate.getAllJobs();

  Assert.assertEquals(State.FAILED, allJobs[0].getState());
  Assert.assertEquals(State.SUCCEEDED, allJobs[1].getState());
  Assert.assertEquals(State.KILLED, allJobs[2].getState());
  Assert.assertEquals(State.FAILED, allJobs[3].getState());
}
 

import org.apache.hadoop.mapreduce.JobStatus; //導入依賴的package包/類
/**
 * To ensure nothing broken after we removed normalization 
 * from the MRAM side
 * @throws Exception
 */
@Test
public void testJobWithNonNormalizedCapabilities() throws Exception {
  if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
    LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
              + " not found. Not running test.");
    return;
  }

  JobConf jobConf = new JobConf(mrCluster.getConfig());
  jobConf.setInt("mapreduce.map.memory.mb", 700);
  jobConf.setInt("mapred.reduce.memory.mb", 1500);

  SleepJob sleepJob = new SleepJob();
  sleepJob.setConf(jobConf);
  Job job = sleepJob.createJob(3, 2, 1000, 1, 500, 1);
  job.setJarByClass(SleepJob.class);
  job.addFileToClassPath(APP_JAR); // The AppMaster jar itself.
  job.submit();
  boolean completed = job.waitForCompletion(true);
  Assert.assertTrue("Job should be completed", completed);
  Assert.assertEquals("Job should be finished successfully", 
                  JobStatus.State.SUCCEEDED, job.getJobState());
}
 

import org.apache.hadoop.mapreduce.JobStatus; //導入依賴的package包/類
/**
 * Method used to display information pertaining to a Single JobQueue
 * registered with the {@link QueueManager}. Display of the Jobs is determine
 * by the boolean
 * 
 * @throws IOException, InterruptedException
 */
private void displayQueueInfo(String queue, boolean showJobs)
    throws IOException, InterruptedException {
  JobQueueInfo jobQueueInfo = jc.getQueueInfo(queue);
  
  if (jobQueueInfo == null) {
    System.out.println("Queue \"" + queue + "\" does not exist.");
    return;
  }
  printJobQueueInfo(jobQueueInfo, new PrintWriter(new OutputStreamWriter(
      System.out, Charsets.UTF_8)));
  if (showJobs && (jobQueueInfo.getChildren() == null ||
      jobQueueInfo.getChildren().size() == 0)) {
    JobStatus[] jobs = jobQueueInfo.getJobStatuses();
    if (jobs == null)
      jobs = new JobStatus[0];
    jc.displayJobList(jobs);
  }
}
 

import org.apache.hadoop.mapreduce.JobStatus; //導入依賴的package包/類
@Private
public void displayJobList(JobStatus[] jobs, PrintWriter writer) {
  writer.println("Total jobs:" + jobs.length);
  writer.printf(headerPattern, "JobId", "State", "StartTime", "UserName",
    "Queue", "Priority", "UsedContainers",
    "RsvdContainers", "UsedMem", "RsvdMem", "NeededMem", "AM info");
  for (JobStatus job : jobs) {
    int numUsedSlots = job.getNumUsedSlots();
    int numReservedSlots = job.getNumReservedSlots();
    int usedMem = job.getUsedMem();
    int rsvdMem = job.getReservedMem();
    int neededMem = job.getNeededMem();
    writer.printf(dataPattern,
        job.getJobID().toString(), job.getState(), job.getStartTime(),
        job.getUsername(), job.getQueue(), 
        job.getPriority().name(),
        numUsedSlots < 0 ? UNAVAILABLE : numUsedSlots,
        numReservedSlots < 0 ? UNAVAILABLE : numReservedSlots,
        usedMem < 0 ? UNAVAILABLE : String.format(memPattern, usedMem),
        rsvdMem < 0 ? UNAVAILABLE : String.format(memPattern, rsvdMem),
        neededMem < 0 ? UNAVAILABLE : String.format(memPattern, neededMem),
        job.getSchedulingInfo());
  }
  writer.flush();
}
 

import org.apache.hadoop.mapreduce.JobStatus; //導入依賴的package包/類
/**
 * Test {@link Statistics.JobStats}.
 */
@Test
@SuppressWarnings("deprecation")
public void testJobStats() throws Exception {
  Job job = new Job() {};
  JobStats stats = new JobStats(1, 2, job);
  assertEquals("Incorrect num-maps", 1, stats.getNoOfMaps());
  assertEquals("Incorrect num-reds", 2, stats.getNoOfReds());
  assertTrue("Incorrect job", job == stats.getJob());
  assertNull("Unexpected job status", stats.getJobStatus());
  
  // add a new status
  JobStatus status = new JobStatus();
  stats.updateJobStatus(status);
  assertNotNull("Missing job status", stats.getJobStatus());
  assertTrue("Incorrect job status", status == stats.getJobStatus());
}
 

import org.apache.hadoop.mapreduce.JobStatus; //導入依賴的package包/類
@Private
public void displayJobList(JobStatus[] jobs, PrintWriter writer) {
  writer.println("Total jobs:" + jobs.length);
  writer.printf(headerPattern, "JobId", "JobName", "State", "StartTime",
    "UserName", "Queue", "Priority", "UsedContainers",
    "RsvdContainers", "UsedMem", "RsvdMem", "NeededMem", "AM info");
  for (JobStatus job : jobs) {
    int numUsedSlots = job.getNumUsedSlots();
    int numReservedSlots = job.getNumReservedSlots();
    int usedMem = job.getUsedMem();
    int rsvdMem = job.getReservedMem();
    int neededMem = job.getNeededMem();
    int jobNameLength = job.getJobName().length();
    writer.printf(dataPattern, job.getJobID().toString(),
        job.getJobName().substring(0, jobNameLength > 20 ? 20 : jobNameLength),
        job.getState(), job.getStartTime(), job.getUsername(),
        job.getQueue(), job.getPriority().name(),
        numUsedSlots < 0 ? UNAVAILABLE : numUsedSlots,
        numReservedSlots < 0 ? UNAVAILABLE : numReservedSlots,
        usedMem < 0 ? UNAVAILABLE : String.format(memPattern, usedMem),
        rsvdMem < 0 ? UNAVAILABLE : String.format(memPattern, rsvdMem),
        neededMem < 0 ? UNAVAILABLE : String.format(memPattern, neededMem),
        job.getSchedulingInfo());
  }
  writer.flush();
}
 

import org.apache.hadoop.mapreduce.JobStatus; //導入依賴的package包/類
public static int getReturnCode(JobStatus.State state) {
    switch (state) {
        case RUNNING:
            return -1;
        case SUCCEEDED:
            return 0;
        case FAILED:
            return 1;
        case PREP:
            return 2;
        case KILLED:
            return 3;
        default:
            return 4;
    }
}
 

import org.apache.hadoop.mapreduce.JobStatus; //導入依賴的package包/類
private static void testJobStats(JobStats stats, int numMaps, int numReds,
                                 JobStatus jStatus, Job job) {
  assertEquals("Incorrect num map tasks", numMaps, stats.getNoOfMaps());
  assertEquals("Incorrect num reduce tasks", numReds, stats.getNoOfReds());
  
  if (job != null) {
    assertNotNull("Missing job", job);
  }
  // check running job
  assertTrue("Incorrect job", job == stats.getJob());
  
  if (jStatus != null) {
    assertNotNull("Missing job status", jStatus);
  }
  // check job stats
  assertTrue("Incorrect job status", jStatus == stats.getJobStatus());
}
 

import org.apache.hadoop.mapreduce.JobStatus; //導入依賴的package包/類
/** {@inheritDoc} */
@Override public JobStatus getJobStatus(JobID jobId) throws IOException, InterruptedException {
    try {
        Long delay = conf.getLong(HadoopJobProperty.JOB_STATUS_POLL_DELAY.propertyName(), -1);

        HadoopJobStatus status;

        if (delay >= 0)
            status = execute(HadoopProtocolJobStatusTask.class, jobId.getJtIdentifier(), jobId.getId(), delay);
        else
            status = execute(HadoopProtocolJobStatusTask.class, jobId.getJtIdentifier(), jobId.getId());

        if (status == null)
            throw new IOException("Job tracker doesn't have any information about the job: " + jobId);

        return processStatus(status);
    }
    catch (GridClientException e) {
        throw new IOException("Failed to get job status: " + jobId, e);
    }
}
 

import org.apache.hadoop.mapreduce.JobStatus; //導入依賴的package包/類
public void readFields(DataInput in) throws IOException {
  jobFile = StringInterner.weakIntern(Text.readString(in));
  taskId = TaskAttemptID.read(in);
  partition = in.readInt();
  numSlotsRequired = in.readInt();
  taskStatus.readFields(in);
  skipRanges.readFields(in);
  currentRecIndexIterator = skipRanges.skipRangeIterator();
  currentRecStartIndex = currentRecIndexIterator.next();
  skipping = in.readBoolean();
  jobCleanup = in.readBoolean();
  if (jobCleanup) {
    jobRunStateForCleanup = 
      WritableUtils.readEnum(in, JobStatus.State.class);
  }
  jobSetup = in.readBoolean();
  writeSkipRecs = in.readBoolean();
  taskCleanup = in.readBoolean();
  if (taskCleanup) {
    setPhase(TaskStatus.Phase.CLEANUP);
  }
  user = StringInterner.weakIntern(Text.readString(in));
  extraData.readFields(in);
}