Java MRApp.waitForState方法代码示例

import org.apache.hadoop.mapreduce.v2.app.MRApp; //导入方法依赖的package包/类
private void testMRAppHistory(MRApp app) throws Exception {
  Configuration conf = new Configuration();
  Job job = app.submit(conf);
  app.waitForState(job, JobState.FAILED);
  Map<TaskId, Task> tasks = job.getTasks();

  Assert.assertEquals("Num tasks is not correct", 1, tasks.size());
  Task task = tasks.values().iterator().next();
  Assert.assertEquals("Task state not correct", TaskState.FAILED, task
      .getReport().getTaskState());
  Map<TaskAttemptId, TaskAttempt> attempts = tasks.values().iterator().next()
      .getAttempts();
  Assert.assertEquals("Num attempts is not correct", 4, attempts.size());

  Iterator<TaskAttempt> it = attempts.values().iterator();
  TaskAttemptReport report = it.next().getReport();
  Assert.assertEquals("Attempt state not correct", TaskAttemptState.FAILED,
      report.getTaskAttemptState());
  Assert.assertEquals("Diagnostic Information is not Correct",
      "Test Diagnostic Event", report.getDiagnosticInfo());
  report = it.next().getReport();
  Assert.assertEquals("Attempt state not correct", TaskAttemptState.FAILED,
      report.getTaskAttemptState());
}
 

import org.apache.hadoop.mapreduce.v2.app.MRApp; //导入方法依赖的package包/类
@Test
public void testMRWebAppSSLDisabled() throws Exception {
  MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true) {
    @Override
    protected ClientService createClientService(AppContext context) {
      return new MRClientService(context);
    }
  };
  Configuration conf = new Configuration();
  // MR is explicitly disabling SSL, even though setting as HTTPS_ONLY
  conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, Policy.HTTPS_ONLY.name());
  Job job = app.submit(conf);

  String hostPort =
      NetUtils.getHostPortString(((MRClientService) app.getClientService())
        .getWebApp().getListenerAddress());
  // http:// should be accessible
  URL httpUrl = new URL("http://" + hostPort);
  HttpURLConnection conn = (HttpURLConnection) httpUrl.openConnection();
  InputStream in = conn.getInputStream();
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  IOUtils.copyBytes(in, out, 1024);
  Assert.assertTrue(out.toString().contains("MapReduce Application"));

  // https:// is not accessible.
  URL httpsUrl = new URL("https://" + hostPort);
  try {
    HttpURLConnection httpsConn =
        (HttpURLConnection) httpsUrl.openConnection();
    httpsConn.getInputStream();
    Assert.fail("https:// is not accessible, expected to fail");
  } catch (Exception e) {
    Assert.assertTrue(e instanceof SSLException);
  }

  app.waitForState(job, JobState.SUCCEEDED);
  app.verifyCompleted();
}
 

import org.apache.hadoop.mapreduce.v2.app.MRApp; //导入方法依赖的package包/类
@Test
public void testMRWebAppRedirection() throws Exception {

  String[] schemePrefix =
      { WebAppUtils.HTTP_PREFIX, WebAppUtils.HTTPS_PREFIX };
  for (String scheme : schemePrefix) {
    MRApp app = new MRApp(2, 2, true, this.getClass().getName(), true) {
      @Override
      protected ClientService createClientService(AppContext context) {
        return new MRClientService(context);
      }
    };
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.PROXY_ADDRESS, "9.9.9.9");
    conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, scheme
      .equals(WebAppUtils.HTTPS_PREFIX) ? Policy.HTTPS_ONLY.name()
        : Policy.HTTP_ONLY.name());
    webProxyBase = "/proxy/" + app.getAppID();
    conf.set("hadoop.http.filter.initializers",
      TestAMFilterInitializer.class.getName());
    Job job = app.submit(conf);
    String hostPort =
        NetUtils.getHostPortString(((MRClientService) app.getClientService())
          .getWebApp().getListenerAddress());
    URL httpUrl = new URL("http://" + hostPort + "/mapreduce");

    HttpURLConnection conn = (HttpURLConnection) httpUrl.openConnection();
    conn.setInstanceFollowRedirects(false);
    conn.connect();
    String expectedURL =
        scheme + conf.get(YarnConfiguration.PROXY_ADDRESS)
            + ProxyUriUtils.getPath(app.getAppID(), "/mapreduce");
    Assert.assertEquals(expectedURL,
      conn.getHeaderField(HttpHeaders.LOCATION));
    Assert.assertEquals(HttpStatus.SC_MOVED_TEMPORARILY,
      conn.getResponseCode());
    app.waitForState(job, JobState.SUCCEEDED);
    app.verifyCompleted();
  }
}
 

import org.apache.hadoop.mapreduce.v2.app.MRApp; //导入方法依赖的package包/类
private void finishTask(DrainDispatcher rmDispatcher, MockNM node,
    MRApp mrApp, Task task) throws Exception {
  TaskAttempt attempt = task.getAttempts().values().iterator().next();
  List<ContainerStatus> contStatus = new ArrayList<ContainerStatus>(1);
  contStatus.add(ContainerStatus.newInstance(attempt.getAssignedContainerID(),
      ContainerState.COMPLETE, "", 0));
  Map<ApplicationId,List<ContainerStatus>> statusUpdate =
      new HashMap<ApplicationId,List<ContainerStatus>>(1);
  statusUpdate.put(mrApp.getAppID(), contStatus);
  node.nodeHeartbeat(statusUpdate, true);
  rmDispatcher.await();
  mrApp.getContext().getEventHandler().handle(
        new TaskAttemptEvent(attempt.getID(), TaskAttemptEventType.TA_DONE));
  mrApp.waitForState(task, TaskState.SUCCEEDED);
}
 

import org.apache.hadoop.mapreduce.v2.app.MRApp; //导入方法依赖的package包/类
/**
 * Verify that all the events are flushed on stopping the HistoryHandler
 * @throws Exception
 */
@Test
public void testEventsFlushOnStop() throws Exception {

  Configuration conf = new Configuration();
  MRApp app = new MRAppWithSpecialHistoryHandler(1, 0, true, this
      .getClass().getName(), true);
  app.submit(conf);
  Job job = app.getContext().getAllJobs().values().iterator().next();
  JobId jobId = job.getID();
  LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
  app.waitForState(job, JobState.SUCCEEDED);

  // make sure all events are flushed
  app.waitForState(Service.STATE.STOPPED);
  /*
   * Use HistoryContext to read logged events and verify the number of
   * completed maps
   */
  HistoryContext context = new JobHistory();
  ((JobHistory) context).init(conf);
  Job parsedJob = context.getJob(jobId);
  Assert.assertEquals("CompletedMaps not correct", 1, parsedJob
      .getCompletedMaps());

  Map<TaskId, Task> tasks = parsedJob.getTasks();
  Assert.assertEquals("No of tasks not correct", 1, tasks.size());
  verifyTask(tasks.values().iterator().next());

  Map<TaskId, Task> maps = parsedJob.getTasks(TaskType.MAP);
  Assert.assertEquals("No of maps not correct", 1, maps.size());

  Assert.assertEquals("Job state not currect", JobState.SUCCEEDED,
      parsedJob.getState());
}
 

import org.apache.hadoop.mapreduce.v2.app.MRApp; //导入方法依赖的package包/类
@Test
public void testAssignedQueue() throws Exception {
  Configuration conf = new Configuration();
  MRApp app = new MRAppWithHistory(2, 1, true, this.getClass().getName(),
      true, "assignedQueue");
  app.submit(conf);
  Job job = app.getContext().getAllJobs().values().iterator().next();
  JobId jobId = job.getID();
  LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
  app.waitForState(job, JobState.SUCCEEDED);
  
  //make sure all events are flushed 
  app.waitForState(Service.STATE.STOPPED);
  /*
   * Use HistoryContext to read logged events and verify the number of 
   * completed maps 
  */
  HistoryContext context = new JobHistory();
  // test start and stop states
  ((JobHistory)context).init(conf);
  ((JobHistory)context).start();
  Assert.assertTrue( context.getStartTime()>0);
  Assert.assertEquals(((JobHistory)context).getServiceState(),Service.STATE.STARTED);

  // get job before stopping JobHistory
  Job parsedJob = context.getJob(jobId);

  // stop JobHistory
  ((JobHistory)context).stop();
  Assert.assertEquals(((JobHistory)context).getServiceState(),Service.STATE.STOPPED);

  Assert.assertEquals("QueueName not correct", "assignedQueue",
      parsedJob.getQueueName());
}
 

import org.apache.hadoop.mapreduce.v2.app.MRApp; //导入方法依赖的package包/类
private static void verifySpeculationMessage(MRApp app, TaskAttempt[] ta)
    throws Exception {
  app.waitForState(ta[0], TaskAttemptState.SUCCEEDED);
  app.waitForState(ta[1], TaskAttemptState.KILLED);
  boolean foundSpecMsg = false;
  for (String msg : ta[1].getDiagnostics()) {
    if (msg.contains("Speculation")) {
      foundSpecMsg = true;
      break;
    }
  }
  Assert.assertTrue("No speculation diagnostics!", foundSpecMsg);
}
 

import org.apache.hadoop.mapreduce.v2.app.MRApp; //导入方法依赖的package包/类
public void verifyMillisCounters(int mapMemMb, int reduceMemMb,
    int minContainerSize) throws Exception {
  Clock actualClock = new SystemClock();
  ControlledClock clock = new ControlledClock(actualClock);
  clock.setTime(10);
  MRApp app =
      new MRApp(1, 1, false, "testSlotMillisCounterUpdate", true, clock);
  Configuration conf = new Configuration();
  conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb);
  conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 
    minContainerSize);
  app.setClusterInfo(new ClusterInfo(Resource.newInstance(10240, 1, 0)));

  Job job = app.submit(conf);
  app.waitForState(job, JobState.RUNNING);
  Map<TaskId, Task> tasks = job.getTasks();
  Assert.assertEquals("Num tasks is not correct", 2, tasks.size());
  Iterator<Task> taskIter = tasks.values().iterator();
  Task mTask = taskIter.next();
  app.waitForState(mTask, TaskState.RUNNING);
  Task rTask = taskIter.next();
  app.waitForState(rTask, TaskState.RUNNING);
  Map<TaskAttemptId, TaskAttempt> mAttempts = mTask.getAttempts();
  Assert.assertEquals("Num attempts is not correct", 1, mAttempts.size());
  Map<TaskAttemptId, TaskAttempt> rAttempts = rTask.getAttempts();
  Assert.assertEquals("Num attempts is not correct", 1, rAttempts.size());
  TaskAttempt mta = mAttempts.values().iterator().next();
  TaskAttempt rta = rAttempts.values().iterator().next();
  app.waitForState(mta, TaskAttemptState.RUNNING);
  app.waitForState(rta, TaskAttemptState.RUNNING);

  clock.setTime(11);
  app.getContext()
      .getEventHandler()
      .handle(new TaskAttemptEvent(mta.getID(), TaskAttemptEventType.TA_DONE));
  app.getContext()
      .getEventHandler()
      .handle(new TaskAttemptEvent(rta.getID(), TaskAttemptEventType.TA_DONE));
  app.waitForState(job, JobState.SUCCEEDED);
  Assert.assertEquals(mta.getFinishTime(), 11);
  Assert.assertEquals(mta.getLaunchTime(), 10);
  Assert.assertEquals(rta.getFinishTime(), 11);
  Assert.assertEquals(rta.getLaunchTime(), 10);
  Counters counters = job.getAllCounters();
  Assert.assertEquals((int) Math.ceil((float) mapMemMb / minContainerSize),
      counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue());
  Assert.assertEquals((int) Math.ceil((float) reduceMemMb / minContainerSize),
      counters.findCounter(JobCounter.SLOTS_MILLIS_REDUCES).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.MILLIS_MAPS).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.MILLIS_REDUCES).getValue());
  Assert.assertEquals(mapMemMb,
      counters.findCounter(JobCounter.MB_MILLIS_MAPS).getValue());
  Assert.assertEquals(reduceMemMb,
      counters.findCounter(JobCounter.MB_MILLIS_REDUCES).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.VCORES_MILLIS_MAPS).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.VCORES_MILLIS_REDUCES).getValue());
}
 

import org.apache.hadoop.mapreduce.v2.app.MRApp; //导入方法依赖的package包/类
@Test(timeout = 15000)
public void testSlowNM() throws Exception {

  conf = new Configuration();
  int maxAttempts = 1;
  conf.setInt(MRJobConfig.MAP_MAX_ATTEMPTS, maxAttempts);
  conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
  // set timeout low for the test
  conf.setInt("yarn.rpc.nm-command-timeout", 3000);
  conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class.getName());
  YarnRPC rpc = YarnRPC.create(conf);
  String bindAddr = "localhost:0";
  InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
  NMTokenSecretManagerInNM tokenSecretManager =
      new NMTokenSecretManagerInNM();
  MasterKey masterKey = Records.newRecord(MasterKey.class);
  masterKey.setBytes(ByteBuffer.wrap("key".getBytes()));
  tokenSecretManager.setMasterKey(masterKey);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    "token");
  server =
      rpc.getServer(ContainerManagementProtocol.class,
        new DummyContainerManager(), addr, conf, tokenSecretManager, 1);
  server.start();

  MRApp app = new MRAppWithSlowNM(tokenSecretManager);

  try {
  Job job = app.submit(conf);
  app.waitForState(job, JobState.RUNNING);

  Map<TaskId, Task> tasks = job.getTasks();
  Assert.assertEquals("Num tasks is not correct", 1, tasks.size());

  Task task = tasks.values().iterator().next();
  app.waitForState(task, TaskState.SCHEDULED);

  Map<TaskAttemptId, TaskAttempt> attempts = tasks.values().iterator()
      .next().getAttempts();
    Assert.assertEquals("Num attempts is not correct", maxAttempts,
        attempts.size());

  TaskAttempt attempt = attempts.values().iterator().next();
    app.waitForInternalState((TaskAttemptImpl) attempt,
        TaskAttemptStateInternal.ASSIGNED);

  app.waitForState(job, JobState.FAILED);

  String diagnostics = attempt.getDiagnostics().toString();
  LOG.info("attempt.getDiagnostics: " + diagnostics);

    Assert.assertTrue(diagnostics.contains("Container launch failed for "
        + "container_0_0000_01_000000 : "));
    Assert
        .assertTrue(diagnostics
            .contains("java.net.SocketTimeoutException: 3000 millis timeout while waiting for channel"));

  } finally {
    server.stop();
  app.stop();
}
}
 

import org.apache.hadoop.mapreduce.v2.app.MRApp; //导入方法依赖的package包/类
private static void verifySpeculationMessage(MRApp app, TaskAttempt[] ta)
    throws Exception {
  app.waitForState(ta[0], TaskAttemptState.SUCCEEDED);
  // The speculative attempt may be not killed before the MR job succeeds.
}
 

import org.apache.hadoop.mapreduce.v2.app.MRApp; //导入方法依赖的package包/类
@Test(timeout = 50000)
public void testScanningOldDirs() throws Exception {
  LOG.info("STARTING testScanningOldDirs");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);
    RackResolver.init(conf);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();
    LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);

    HistoryFileManagerForTest hfm = new HistoryFileManagerForTest();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    Assert.assertNotNull("Unable to locate job history", fileInfo);

    // force the manager to "forget" the job
    hfm.deleteJobFromJobListCache(fileInfo);
    final int msecPerSleep = 10;
    int msecToSleep = 10 * 1000;
    while (fileInfo.isMovePending() && msecToSleep > 0) {
      Assert.assertTrue(!fileInfo.didMoveFail());
      msecToSleep -= msecPerSleep;
      Thread.sleep(msecPerSleep);
    }
    Assert.assertTrue("Timeout waiting for history move", msecToSleep > 0);

    fileInfo = hfm.getFileInfo(jobId);
    hfm.stop();
    Assert.assertNotNull("Unable to locate old job history", fileInfo);
    Assert.assertTrue("HistoryFileManager not shutdown properly",
        hfm.moveToDoneExecutor.isTerminated());
  } finally {
    LOG.info("FINISHED testScanningOldDirs");
  }
}
 

import org.apache.hadoop.mapreduce.v2.app.MRApp; //导入方法依赖的package包/类
/**
 * test clean old history files. Files should be deleted after 1 week by
 * default.
 */
@Test(timeout = 15000)
public void testDeleteFileInfo() throws Exception {
  LOG.info("STARTING testDeleteFileInfo");
  try {
    Configuration conf = new Configuration();
    conf.setClass(
        NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
        MyResolver.class, DNSToSwitchMapping.class);

    RackResolver.init(conf);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(conf);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    JobId jobId = job.getID();

    app.waitForState(job, JobState.SUCCEEDED);

    // make sure all events are flushed
    app.waitForState(Service.STATE.STOPPED);
    HistoryFileManager hfm = new HistoryFileManager();
    hfm.init(conf);
    HistoryFileInfo fileInfo = hfm.getFileInfo(jobId);
    hfm.initExisting();
    // wait for move files form the done_intermediate directory to the gone
    // directory
    while (fileInfo.isMovePending()) {
      Thread.sleep(300);
    }

    Assert.assertNotNull(hfm.jobListCache.values());

    // try to remove fileInfo
    hfm.clean();
    // check that fileInfo does not deleted
    Assert.assertFalse(fileInfo.isDeleted());
    // correct live time
    hfm.setMaxHistoryAge(-1);
    hfm.clean();
    hfm.stop();
    Assert.assertTrue("Thread pool shutdown",
        hfm.moveToDoneExecutor.isTerminated());
    // should be deleted !
    Assert.assertTrue("file should be deleted ", fileInfo.isDeleted());

  } finally {
    LOG.info("FINISHED testDeleteFileInfo");
  }
}
 

import org.apache.hadoop.mapreduce.v2.app.MRApp; //导入方法依赖的package包/类
/**
 * Simple test some methods of JobHistory
 */
@Test(timeout = 20000)
public void testJobHistoryMethods() throws Exception {
  LOG.info("STARTING testJobHistoryMethods");
  try {
    Configuration configuration = new Configuration();
    configuration
        .setClass(
            NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
            MyResolver.class, DNSToSwitchMapping.class);

    RackResolver.init(configuration);
    MRApp app = new MRAppWithHistory(1, 1, true, this.getClass().getName(),
        true);
    app.submit(configuration);
    Job job = app.getContext().getAllJobs().values().iterator().next();
    app.waitForState(job, JobState.SUCCEEDED);

    JobHistory jobHistory = new JobHistory();
    jobHistory.init(configuration);
    // Method getAllJobs
    Assert.assertEquals(1, jobHistory.getAllJobs().size());
    // and with ApplicationId
    Assert.assertEquals(1, jobHistory.getAllJobs(app.getAppID()).size());

    JobsInfo jobsinfo = jobHistory.getPartialJobs(0L, 10L, null, "default",
        0L, System.currentTimeMillis() + 1, 0L,
        System.currentTimeMillis() + 1, JobState.SUCCEEDED);

    Assert.assertEquals(1, jobsinfo.getJobs().size());
    Assert.assertNotNull(jobHistory.getApplicationAttemptId());
    // test Application Id
    Assert.assertEquals("application_0_0000", jobHistory.getApplicationID()
        .toString());
    Assert
        .assertEquals("Job History Server", jobHistory.getApplicationName());
    // method does not work
    Assert.assertNull(jobHistory.getEventHandler());
    // method does not work
    Assert.assertNull(jobHistory.getClock());
    // method does not work
    Assert.assertNull(jobHistory.getClusterInfo());

  } finally {
    LOG.info("FINISHED testJobHistoryMethods");
  }
}
 

import org.apache.hadoop.mapreduce.v2.app.MRApp; //导入方法依赖的package包/类
@Test
public void testHistoryEvents() throws Exception {
  Configuration conf = new Configuration();
  MRApp app = new MRAppWithHistory(2, 1, true, this.getClass().getName(), true);
  app.submit(conf);
  Job job = app.getContext().getAllJobs().values().iterator().next();
  JobId jobId = job.getID();
  LOG.info("JOBID is " + TypeConverter.fromYarn(jobId).toString());
  app.waitForState(job, JobState.SUCCEEDED);
  
  //make sure all events are flushed 
  app.waitForState(Service.STATE.STOPPED);
  /*
   * Use HistoryContext to read logged events and verify the number of 
   * completed maps 
  */
  HistoryContext context = new JobHistory();
  // test start and stop states
  ((JobHistory)context).init(conf);
  ((JobHistory)context).start();
  Assert.assertTrue( context.getStartTime()>0);
  Assert.assertEquals(((JobHistory)context).getServiceState(),Service.STATE.STARTED);

  // get job before stopping JobHistory
  Job parsedJob = context.getJob(jobId);

  // stop JobHistory
  ((JobHistory)context).stop();
  Assert.assertEquals(((JobHistory)context).getServiceState(),Service.STATE.STOPPED);


  Assert.assertEquals("CompletedMaps not correct", 2,
      parsedJob.getCompletedMaps());
  Assert.assertEquals(System.getProperty("user.name"), parsedJob.getUserName());
  
  Map<TaskId, Task> tasks = parsedJob.getTasks();
  Assert.assertEquals("No of tasks not correct", 3, tasks.size());
  for (Task task : tasks.values()) {
    verifyTask(task);
  }
  
  Map<TaskId, Task> maps = parsedJob.getTasks(TaskType.MAP);
  Assert.assertEquals("No of maps not correct", 2, maps.size());
  
  Map<TaskId, Task> reduces = parsedJob.getTasks(TaskType.REDUCE);
  Assert.assertEquals("No of reduces not correct", 1, reduces.size());
  
  
  Assert.assertEquals("CompletedReduce not correct", 1,
      parsedJob.getCompletedReduces());
  
  Assert.assertEquals("Job state not currect", JobState.SUCCEEDED,
      parsedJob.getState());
}
 

import org.apache.hadoop.mapreduce.v2.app.MRApp; //导入方法依赖的package包/类
public void verifyMillisCounters(int mapMemMb, int reduceMemMb,
    int minContainerSize) throws Exception {
  Clock actualClock = new SystemClock();
  ControlledClock clock = new ControlledClock(actualClock);
  clock.setTime(10);
  MRApp app =
      new MRApp(1, 1, false, "testSlotMillisCounterUpdate", true, clock);
  Configuration conf = new Configuration();
  conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemMb);
  conf.setInt(MRJobConfig.REDUCE_MEMORY_MB, reduceMemMb);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 
    minContainerSize);
  app.setClusterInfo(new ClusterInfo(Resource.newInstance(10240, 1)));

  Job job = app.submit(conf);
  app.waitForState(job, JobState.RUNNING);
  Map<TaskId, Task> tasks = job.getTasks();
  Assert.assertEquals("Num tasks is not correct", 2, tasks.size());
  Iterator<Task> taskIter = tasks.values().iterator();
  Task mTask = taskIter.next();
  app.waitForState(mTask, TaskState.RUNNING);
  Task rTask = taskIter.next();
  app.waitForState(rTask, TaskState.RUNNING);
  Map<TaskAttemptId, TaskAttempt> mAttempts = mTask.getAttempts();
  Assert.assertEquals("Num attempts is not correct", 1, mAttempts.size());
  Map<TaskAttemptId, TaskAttempt> rAttempts = rTask.getAttempts();
  Assert.assertEquals("Num attempts is not correct", 1, rAttempts.size());
  TaskAttempt mta = mAttempts.values().iterator().next();
  TaskAttempt rta = rAttempts.values().iterator().next();
  app.waitForState(mta, TaskAttemptState.RUNNING);
  app.waitForState(rta, TaskAttemptState.RUNNING);

  clock.setTime(11);
  app.getContext()
      .getEventHandler()
      .handle(new TaskAttemptEvent(mta.getID(), TaskAttemptEventType.TA_DONE));
  app.getContext()
      .getEventHandler()
      .handle(new TaskAttemptEvent(rta.getID(), TaskAttemptEventType.TA_DONE));
  app.waitForState(job, JobState.SUCCEEDED);
  Assert.assertEquals(mta.getFinishTime(), 11);
  Assert.assertEquals(mta.getLaunchTime(), 10);
  Assert.assertEquals(rta.getFinishTime(), 11);
  Assert.assertEquals(rta.getLaunchTime(), 10);
  Counters counters = job.getAllCounters();
  Assert.assertEquals((int) Math.ceil((float) mapMemMb / minContainerSize),
      counters.findCounter(JobCounter.SLOTS_MILLIS_MAPS).getValue());
  Assert.assertEquals((int) Math.ceil((float) reduceMemMb / minContainerSize),
      counters.findCounter(JobCounter.SLOTS_MILLIS_REDUCES).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.MILLIS_MAPS).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.MILLIS_REDUCES).getValue());
  Assert.assertEquals(mapMemMb,
      counters.findCounter(JobCounter.MB_MILLIS_MAPS).getValue());
  Assert.assertEquals(reduceMemMb,
      counters.findCounter(JobCounter.MB_MILLIS_REDUCES).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.VCORES_MILLIS_MAPS).getValue());
  Assert.assertEquals(1,
      counters.findCounter(JobCounter.VCORES_MILLIS_REDUCES).getValue());
}