Java JobId.setAppId方法代码示例

import org.apache.hadoop.mapreduce.v2.api.records.JobId; //导入方法依赖的package包/类
@Test
public void testDeletionofStaging() throws IOException {
  conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
  fs = mock(FileSystem.class);
  when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
  //Staging Dir exists
  String user = UserGroupInformation.getCurrentUser().getShortUserName();
  Path stagingDir = MRApps.getStagingAreaDir(conf, user);
  when(fs.exists(stagingDir)).thenReturn(true);
  ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
     0);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
  JobId jobid = recordFactory.newRecordInstance(JobId.class);
  jobid.setAppId(appId);
  ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
  Assert.assertTrue(MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS > 1);
  MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc,
      JobStateInternal.RUNNING, MRJobConfig.DEFAULT_MR_AM_MAX_ATTEMPTS);
  appMaster.init(conf);
  appMaster.start();
  appMaster.shutDownJob();
  //test whether notifyIsLastAMRetry called
  Assert.assertEquals(true, ((TestMRApp)appMaster).getTestIsLastAMRetry());
  verify(fs).delete(stagingJobPath, true);
}
 

import org.apache.hadoop.mapreduce.v2.api.records.JobId; //导入方法依赖的package包/类
@Test (timeout = 30000)
public void testDeletionofStagingOnKill() throws IOException {
  conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
  fs = mock(FileSystem.class);
  when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
  //Staging Dir exists
  String user = UserGroupInformation.getCurrentUser().getShortUserName();
  Path stagingDir = MRApps.getStagingAreaDir(conf, user);
  when(fs.exists(stagingDir)).thenReturn(true);
  ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
      0);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 0);
  JobId jobid = recordFactory.newRecordInstance(JobId.class);
  jobid.setAppId(appId);
  ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
  MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc);
  appMaster.init(conf);
  //simulate the process being killed
  MRAppMaster.MRAppMasterShutdownHook hook = 
    new MRAppMaster.MRAppMasterShutdownHook(appMaster);
  hook.run();
  verify(fs, times(0)).delete(stagingJobPath, true);
}
 

import org.apache.hadoop.mapreduce.v2.api.records.JobId; //导入方法依赖的package包/类
public void testDeletionofStagingOnKillLastTry() throws IOException {
  conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
  fs = mock(FileSystem.class);
  when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
  //Staging Dir exists
  String user = UserGroupInformation.getCurrentUser().getShortUserName();
  Path stagingDir = MRApps.getStagingAreaDir(conf, user);
  when(fs.exists(stagingDir)).thenReturn(true);
  ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(),
      0);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
  JobId jobid = recordFactory.newRecordInstance(JobId.class);
  jobid.setAppId(appId);
  ContainerAllocator mockAlloc = mock(ContainerAllocator.class);
  MRAppMaster appMaster = new TestMRApp(attemptId, mockAlloc); //no retry
  appMaster.init(conf);
  assertTrue("appMaster.isLastAMRetry() is false", appMaster.isLastAMRetry());
  //simulate the process being killed
  MRAppMaster.MRAppMasterShutdownHook hook = 
    new MRAppMaster.MRAppMasterShutdownHook(appMaster);
  hook.run();
  assertTrue("MRAppMaster isn't stopped",
             appMaster.isInState(Service.STATE.STOPPED));
  verify(fs).delete(stagingJobPath, true);
}
 

import org.apache.hadoop.mapreduce.v2.api.records.JobId; //导入方法依赖的package包/类
@Test
public void testFromYarnJobReport() throws Exception {
  int jobStartTime = 612354;
  int jobFinishTime = 612355;
  JobState state = JobState.RUNNING;
  JobId jobId = Records.newRecord(JobId.class);
  JobReport jobReport = Records.newRecord(JobReport.class);
  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  jobId.setAppId(applicationId);
  jobId.setId(0);    
  jobReport.setJobId(jobId);
  jobReport.setJobState(state);
  jobReport.setStartTime(jobStartTime);
  jobReport.setFinishTime(jobFinishTime);
  jobReport.setUser("TestTypeConverter-user");    
  JobStatus jobStatus = TypeConverter.fromYarn(jobReport, "dummy-jobfile");
  Assert.assertEquals(jobStartTime, jobStatus.getStartTime());
  Assert.assertEquals(jobFinishTime, jobStatus.getFinishTime());    
  Assert.assertEquals(state.toString(), jobStatus.getState().toString());
}
 

import org.apache.hadoop.mapreduce.v2.api.records.JobId; //导入方法依赖的package包/类
private Job getJob() {
  Job job = mock(Job.class);

  JobId jobId = new JobIdPBImpl();

  ApplicationId appId = ApplicationIdPBImpl.newInstance(System.currentTimeMillis(),4);
  jobId.setAppId(appId);
  jobId.setId(1);
  when(job.getID()).thenReturn(jobId);

  JobReport report = mock(JobReport.class);
  when(report.getStartTime()).thenReturn(100010L);
  when(report.getFinishTime()).thenReturn(100015L);

  when(job.getReport()).thenReturn(report);
  when(job.getName()).thenReturn("JobName");
  when(job.getUserName()).thenReturn("UserName");
  when(job.getQueueName()).thenReturn("QueueName");
  when(job.getState()).thenReturn(JobState.SUCCEEDED);
  when(job.getTotalMaps()).thenReturn(3);
  when(job.getCompletedMaps()).thenReturn(2);
  when(job.getTotalReduces()).thenReturn(2);
  when(job.getCompletedReduces()).thenReturn(1);
  when(job.getCompletedReduces()).thenReturn(1);
  return job;
}
 

import org.apache.hadoop.mapreduce.v2.api.records.JobId; //导入方法依赖的package包/类
private Task getTask(long timestamp) {
  
  JobId jobId = new JobIdPBImpl();
  jobId.setId(0);
  jobId.setAppId(ApplicationIdPBImpl.newInstance(timestamp,1));

  TaskId taskId = new TaskIdPBImpl();
  taskId.setId(0);
  taskId.setTaskType(TaskType.REDUCE);
  taskId.setJobId(jobId);
  Task task = mock(Task.class);
  when(task.getID()).thenReturn(taskId);
  TaskReport report = mock(TaskReport.class);
  when(report.getProgress()).thenReturn(0.7f);
  when(report.getTaskState()).thenReturn(TaskState.SUCCEEDED);
  when(report.getStartTime()).thenReturn(100001L);
  when(report.getFinishTime()).thenReturn(100011L);

  when(task.getReport()).thenReturn(report);
  when(task.getType()).thenReturn(TaskType.REDUCE);
  return task;
}
 

import org.apache.hadoop.mapreduce.v2.api.records.JobId; //导入方法依赖的package包/类
@SuppressWarnings("resource")
private void testDeletionofStagingOnUnregistrationFailure(
    int maxAttempts, boolean shouldHaveDeleted) throws IOException {
  conf.set(MRJobConfig.MAPREDUCE_JOB_DIR, stagingJobDir);
  fs = mock(FileSystem.class);
  when(fs.delete(any(Path.class), anyBoolean())).thenReturn(true);
  //Staging Dir exists
  String user = UserGroupInformation.getCurrentUser().getShortUserName();
  Path stagingDir = MRApps.getStagingAreaDir(conf, user);
  when(fs.exists(stagingDir)).thenReturn(true);
  ApplicationId appId = ApplicationId.newInstance(0, 1);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
  JobId jobid = recordFactory.newRecordInstance(JobId.class);
  jobid.setAppId(appId);
  TestMRApp appMaster = new TestMRApp(attemptId, null,
      JobStateInternal.RUNNING, maxAttempts);
  appMaster.crushUnregistration = true;
  appMaster.init(conf);
  appMaster.start();
  appMaster.shutDownJob();
  ((RunningAppContext) appMaster.getContext()).resetIsLastAMRetry();
  if (shouldHaveDeleted) {
    Assert.assertEquals(new Boolean(true), appMaster.isLastAMRetry());
    verify(fs).delete(stagingJobPath, true);
  } else {
    Assert.assertEquals(new Boolean(false), appMaster.isLastAMRetry());
    verify(fs, never()).delete(stagingJobPath, true);
  }
}
 

import org.apache.hadoop.mapreduce.v2.api.records.JobId; //导入方法依赖的package包/类
public static JobId toYarn(org.apache.hadoop.mapreduce.JobID id) {
  JobId jobId = recordFactory.newRecordInstance(JobId.class);
  jobId.setId(id.getId()); //currently there is 1-1 mapping between appid and jobid

  ApplicationId appId = ApplicationId.newInstance(
      toClusterTimeStamp(id.getJtIdentifier()), id.getId());
  jobId.setAppId(appId);
  return jobId;
}
 

import org.apache.hadoop.mapreduce.v2.api.records.JobId; //导入方法依赖的package包/类
/**
 * Test rendering for TasksBlock
 */
@Test
public void testTasksBlock() throws Exception {

  ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 1);
  JobId jobId = new JobIdPBImpl();
  jobId.setId(0);
  jobId.setAppId(appId);

  TaskId taskId = new TaskIdPBImpl();
  taskId.setId(0);
  taskId.setTaskType(TaskType.MAP);
  taskId.setJobId(jobId);
  Task task = mock(Task.class);
  when(task.getID()).thenReturn(taskId);
  TaskReport report = mock(TaskReport.class);
  when(report.getProgress()).thenReturn(0.7f);
  when(report.getTaskState()).thenReturn(TaskState.SUCCEEDED);
  when(report.getStartTime()).thenReturn(100001L);
  when(report.getFinishTime()).thenReturn(100011L);
  when(report.getStatus()).thenReturn("Dummy Status \n*");


  when(task.getReport()).thenReturn(report);
  when(task.getType()).thenReturn(TaskType.MAP);


  Map<TaskId, Task> tasks = new HashMap<TaskId, Task>();
  tasks.put(taskId, task);
  AppContext ctx = mock(AppContext.class);
  Job job = mock(Job.class);
  when(job.getTasks()).thenReturn(tasks);


  App app = new App(ctx);
  app.setJob(job);
  TasksBlockForTest taskBlock = new TasksBlockForTest(app);
  taskBlock.addParameter(AMParams.TASK_TYPE, "m");

  PrintWriter pWriter = new PrintWriter(data);
  Block html = new BlockForTest(new HtmlBlockForTest(), pWriter, 0, false);

  taskBlock.render(html);
  pWriter.flush();
  assertTrue(data.toString().contains("task_0_0001_m_000000"));
  assertTrue(data.toString().contains("70.00"));
  assertTrue(data.toString().contains("SUCCEEDED"));
  assertTrue(data.toString().contains("100001"));
  assertTrue(data.toString().contains("100011"));
  assertFalse(data.toString().contains("Dummy Status \n*"));
  assertTrue(data.toString().contains("Dummy Status \\n*"));


}
 

import org.apache.hadoop.mapreduce.v2.api.records.JobId; //导入方法依赖的package包/类
public static JobId newJobID(ApplicationId appID, int i) {
  JobId id = Records.newRecord(JobId.class);
  id.setAppId(appID);
  id.setId(i);
  return id;
}
 

import org.apache.hadoop.mapreduce.v2.api.records.JobId; //导入方法依赖的package包/类
public static JobId newJobId(ApplicationId appId, int id) {
  JobId jobId = Records.newRecord(JobId.class);
  jobId.setAppId(appId);
  jobId.setId(id);
  return jobId;
}
 

import org.apache.hadoop.mapreduce.v2.api.records.JobId; //导入方法依赖的package包/类
@Test (timeout = 120000)
public void testJobIDtoString() {
  JobId jid = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(JobId.class);
  jid.setAppId(ApplicationId.newInstance(0, 0));
  assertEquals("job_0_0000", MRApps.toString(jid));
}
 

import org.apache.hadoop.mapreduce.v2.api.records.JobId; //导入方法依赖的package包/类
private static JobId createJobId(int id) {
  JobId jobId = new JobIdPBImpl();
  jobId.setId(id);
  jobId.setAppId(ApplicationIdPBImpl.newInstance(0, id));
  return jobId;
}