Java Job.getID方法代码示例

import org.apache.hadoop.mapreduce.v2.app.job.Job; //导入方法依赖的package包/类
private void checkAccess(Job job, JobACL jobOperation)
    throws IOException {

  UserGroupInformation callerUGI;
  callerUGI = UserGroupInformation.getCurrentUser();

  if (!job.checkAccess(callerUGI, jobOperation)) {
    throw new IOException(new AccessControlException("User "
        + callerUGI.getShortUserName() + " cannot perform operation "
        + jobOperation.name() + " on " + job.getID()));
  }
}
 

import org.apache.hadoop.mapreduce.v2.app.job.Job; //导入方法依赖的package包/类
/**
 * Verify that all the events are flushed on stopping the HistoryHandler
 * @throws Exception
 */
@Test
public void testEventsFlushOnStop() throws Exception {

  Configuration conf = new Configuration();
  MRApp app = new MRAppWithSpecialHistoryHandler(1, 0, true, this
      .getClass().getName(), true);
  app.submit(conf);
  Job job = app.getContext().getAllJobs().values().iterator().next();
  JobId jobId = job.getID();
  LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
  app.waitForState(job, JobState.SUCCEEDED);

  // make sure all events are flushed
  app.waitForState(Service.STATE.STOPPED);
  /*
   * Use HistoryContext to read logged events and verify the number of
   * completed maps
   */
  HistoryContext context = new JobHistory();
  ((JobHistory) context).init(conf);
  Job parsedJob = context.getJob(jobId);
  Assert.assertEquals("CompletedMaps not correct", 1, parsedJob
      .getCompletedMaps());

  Map<TaskId, Task> tasks = parsedJob.getTasks();
  Assert.assertEquals("No of tasks not correct", 1, tasks.size());
  verifyTask(tasks.values().iterator().next());

  Map<TaskId, Task> maps = parsedJob.getTasks(TaskType.MAP);
  Assert.assertEquals("No of maps not correct", 1, maps.size());

  Assert.assertEquals("Job state not currect", JobState.SUCCEEDED,
      parsedJob.getState());
}
 

import org.apache.hadoop.mapreduce.v2.app.job.Job; //导入方法依赖的package包/类
@Test
public void testAssignedQueue() throws Exception {
  Configuration conf = new Configuration();
  MRApp app = new MRAppWithHistory(2, 1, true, this.getClass().getName(),
      true, "assignedQueue");
  app.submit(conf);
  Job job = app.getContext().getAllJobs().values().iterator().next();
  JobId jobId = job.getID();
  LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
  app.waitForState(job, JobState.SUCCEEDED);
  
  //make sure all events are flushed 
  app.waitForState(Service.STATE.STOPPED);
  /*
   * Use HistoryContext to read logged events and verify the number of 
   * completed maps 
  */
  HistoryContext context = new JobHistory();
  // test start and stop states
  ((JobHistory)context).init(conf);
  ((JobHistory)context).start();
  Assert.assertTrue( context.getStartTime()>0);
  Assert.assertEquals(((JobHistory)context).getServiceState(),Service.STATE.STARTED);

  // get job before stopping JobHistory
  Job parsedJob = context.getJob(jobId);

  // stop JobHistory
  ((JobHistory)context).stop();
  Assert.assertEquals(((JobHistory)context).getServiceState(),Service.STATE.STOPPED);

  Assert.assertEquals("QueueName not correct", "assignedQueue",
      parsedJob.getQueueName());
}
 

import org.apache.hadoop.mapreduce.v2.app.job.Job; //导入方法依赖的package包/类
@Test(timeout = 50000)
public void testScanningOldDirs() throws Exception {
  LOG.info("STARTING testScanningOldDirs");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);
    RackResolver.init(conf);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);

    HistoryFileManagerForTest hfm = new HistoryFileManagerForTest();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    Assert.assertNotNull("Unable to locate job history", fileInfo);

    // force the manager to "forget" the job
    hfm.deleteJobFromJobListCache(fileInfo);
    final int msecPerSleep = 10;
    int msecToSleep = 10 * 1000;
    while (fileInfo.isMovePending() && msecToSleep > 0) {
      Assert.assertTrue(!fileInfo.didMoveFail());
      msecToSleep -= msecPerSleep;
      Thread.sleep(msecPerSleep);
    }
    Assert.assertTrue("Timeout waiting for history move", msecToSleep > 0);

    fileInfo = hfm.getFileInfo(jobId);
    hfm.stop();
    Assert.assertNotNull("Unable to locate old job history", fileInfo);
    Assert.assertTrue("HistoryFileManager not shutdown properly",
        hfm.moveToDoneExecutor.isTerminated());
  } finally {
    LOG.info("FINISHED testScanningOldDirs");
  }
}
 

import org.apache.hadoop.mapreduce.v2.app.job.Job; //导入方法依赖的package包/类
/**
 * test clean old history files. Files should be deleted after 1 week by
 * default.
 */
@Test(timeout = 15000)
public void testDeleteFileInfo() throws Exception {
  LOG.info("STARTING testDeleteFileInfo");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);

    RackResolver.init(conf);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();

    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);
    HistoryFileManager hfm = new HistoryFileManager();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    hfm.initExisting();
    // wait for move files form the done_intermediate directory to the gone
    // directory
    while (fileInfo.isMovePending()) {
      Thread.sleep(300);
    }

    Assert.assertNotNull(hfm.jobListCache.values());

    // try to remove fileInfo
    hfm.clean();
    // check that fileInfo does not deleted
    Assert.assertFalse(fileInfo.isDeleted());
    // correct live time
    hfm.setMaxHistoryAge(-1);
    hfm.clean();
    hfm.stop();
    Assert.assertTrue("Thread pool shutdown",
        hfm.moveToDoneExecutor.isTerminated());
    // should be deleted !
    Assert.assertTrue("file should be deleted ", fileInfo.isDeleted());

  } finally {
    LOG.info("FINISHED testDeleteFileInfo");
  }
}
 

import org.apache.hadoop.mapreduce.v2.app.job.Job; //导入方法依赖的package包/类
@Test
public void testHistoryEvents() throws Exception {
  Configuration conf = new Configuration();
  MRApp app = new MRAppWithHistory(2, 1, true, this.getClass().getName(), true);
  app.submit(conf);
  Job job = app.getContext().getAllJobs().values().iterator().next();
  JobId jobId = job.getID();
  LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
  app.waitForState(job, JobState.SUCCEEDED);
  
  //make sure all events are flushed 
  app.waitForState(Service.STATE.STOPPED);
  /*
   * Use HistoryContext to read logged events and verify the number of 
   * completed maps 
  */
  HistoryContext context = new JobHistory();
  // test start and stop states
  ((JobHistory)context).init(conf);
  ((JobHistory)context).start();
  Assert.assertTrue( context.getStartTime()>0);
  Assert.assertEquals(((JobHistory)context).getServiceState(),Service.STATE.STARTED);

  // get job before stopping JobHistory
  Job parsedJob = context.getJob(jobId);

  // stop JobHistory
  ((JobHistory)context).stop();
  Assert.assertEquals(((JobHistory)context).getServiceState(),Service.STATE.STOPPED);


  Assert.assertEquals("CompletedMaps not correct", 2,
      parsedJob.getCompletedMaps());
  Assert.assertEquals(System.getProperty("user.name"), parsedJob.getUserName());
  
  Map<TaskId, Task> tasks = parsedJob.getTasks();
  Assert.assertEquals("No of tasks not correct", 3, tasks.size());
  for (Task task : tasks.values()) {
    verifyTask(task);
  }
  
  Map<TaskId, Task> maps = parsedJob.getTasks(TaskType.MAP);
  Assert.assertEquals("No of maps not correct", 2, maps.size());
  
  Map<TaskId, Task> reduces = parsedJob.getTasks(TaskType.REDUCE);
  Assert.assertEquals("No of reduces not correct", 1, reduces.size());
  
  
  Assert.assertEquals("CompletedReduce not correct", 1,
      parsedJob.getCompletedReduces());
  
  Assert.assertEquals("Job state not currect", JobState.SUCCEEDED,
      parsedJob.getState());
}
 

import org.apache.hadoop.mapreduce.v2.app.job.Job; //导入方法依赖的package包/类
public MockCompletedJob(Job job) throws IOException {
  super(new Configuration(), job.getID(), null, true, job.getUserName(),
      null, null);
  this.job = job;
}