Java ConverterUtils.toApplicationAttemptId方法代码示例

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
@Override
protected void modifyAppState() throws Exception {
  // imitate appAttemptFile1 is still .new, but old one is deleted
  String appAttemptIdStr1 = "appattempt_1352994193343_0001_000001";
  ApplicationAttemptId attemptId1 =
      ConverterUtils.toApplicationAttemptId(appAttemptIdStr1);
  Path appDir =
          fsTester.store.getAppDir(attemptId1.getApplicationId().toString());
  Path appAttemptFile1 =
      new Path(appDir, attemptId1.toString() + ".new");
  FileSystemRMStateStore fileSystemRMStateStore =
      (FileSystemRMStateStore) fsTester.getRMStateStore();
  fileSystemRMStateStore.renameFile(appAttemptFile1,
          new Path(appAttemptFile1.getParent(),
                  appAttemptFile1.getName() + ".new"));
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
private static CommitterEventHandler createCommitterEventHandler(
    Dispatcher dispatcher, OutputCommitter committer) {
  final SystemClock clock = new SystemClock();
  AppContext appContext = mock(AppContext.class);
  when(appContext.getEventHandler()).thenReturn(
      dispatcher.getEventHandler());
  when(appContext.getClock()).thenReturn(clock);
  RMHeartbeatHandler heartbeatHandler = new RMHeartbeatHandler() {
    @Override
    public long getLastHeartbeatTime() {
      return clock.getTime();
    }
    @Override
    public void runOnNextHeartbeat(Runnable callback) {
      callback.run();
    }
  };
  ApplicationAttemptId id = 
    ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
  when(appContext.getApplicationID()).thenReturn(id.getApplicationId());
  when(appContext.getApplicationAttemptId()).thenReturn(id);
  CommitterEventHandler handler =
      new CommitterEventHandler(appContext, committer, heartbeatHandler);
  dispatcher.register(CommitterEventType.class, handler);
  return handler;
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
@Test
public void testMRAppMasterForDifferentUser() throws IOException,
    InterruptedException {
  String applicationAttemptIdStr = "appattempt_1317529182569_0004_000001";
  String containerIdStr = "container_1317529182569_0004_000001_1";
  
  String userName = "TestAppMasterUser";
  ApplicationAttemptId applicationAttemptId = ConverterUtils
      .toApplicationAttemptId(applicationAttemptIdStr);
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  MRAppMasterTest appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
          System.currentTimeMillis());
  JobConf conf = new JobConf();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
  Path userPath = new Path(stagingDir, userName);
  Path userStagingPath = new Path(userPath, ".staging");
  assertEquals(userStagingPath.toString(),
    appMaster.stagingDirPath.toString());
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
protected static ApplicationAttemptId parseApplicationAttemptId(
    String appAttemptId) {
  if (appAttemptId == null || appAttemptId.isEmpty()) {
    throw new NotFoundException("appAttemptId, " + appAttemptId
        + ", is empty or null");
  }
  ApplicationAttemptId aaid =
      ConverterUtils.toApplicationAttemptId(appAttemptId);
  if (aaid == null) {
    throw new NotFoundException("appAttemptId is null");
  }
  return aaid;
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
private ApplicationAttemptStateData createAttemptState(String itemName,
    byte[] data) throws IOException {
  ApplicationAttemptId attemptId =
      ConverterUtils.toApplicationAttemptId(itemName);
  ApplicationAttemptStateDataPBImpl attemptState =
      new ApplicationAttemptStateDataPBImpl(
          ApplicationAttemptStateDataProto.parseFrom(data));
  if (!attemptId.equals(attemptState.getAttemptId())) {
    throw new YarnRuntimeException("The database entry for " + attemptId
        + " contains data for " + attemptState.getAttemptId());
  }
  return attemptState;
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
@Test
public void testDuplicateRMAppDeletion() throws Exception {
  TestZKRMStateStoreTester zkTester = new TestZKRMStateStoreTester();
  long submitTime = System.currentTimeMillis();
  long startTime = System.currentTimeMillis() + 1234;
  RMStateStore store = zkTester.getRMStateStore();
  TestDispatcher dispatcher = new TestDispatcher();
  store.setRMDispatcher(dispatcher);

  ApplicationAttemptId attemptIdRemoved = ConverterUtils
      .toApplicationAttemptId("appattempt_1352994193343_0002_000001");
  ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId();
  storeApp(store, appIdRemoved, submitTime, startTime);
  storeAttempt(store, attemptIdRemoved,
      "container_1352994193343_0002_01_000001", null, null, dispatcher);

  ApplicationSubmissionContext context =
      new ApplicationSubmissionContextPBImpl();
  context.setApplicationId(appIdRemoved);
  ApplicationStateData appStateRemoved =
      ApplicationStateData.newInstance(
          submitTime, startTime, context, "user1");
  appStateRemoved.attempts.put(attemptIdRemoved, null);
  store.removeApplicationStateInternal(appStateRemoved);
  try {
    store.removeApplicationStateInternal(appStateRemoved);
  } catch (KeeperException.NoNodeException nne) {
    Assert.fail("NoNodeException should not happen.");
  }
  store.close();
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
@Test(timeout = 60000)
public void testFSRMStateStore() throws Exception {
  HdfsConfiguration conf = new HdfsConfiguration();
  MiniDFSCluster cluster =
          new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
  try {
    fsTester = new TestFSRMStateStoreTester(cluster, false);
    // If the state store is FileSystemRMStateStore then add corrupted entry.
    // It should discard the entry and remove it from file system.
    FSDataOutputStream fsOut = null;
    FileSystemRMStateStore fileSystemRMStateStore =
            (FileSystemRMStateStore) fsTester.getRMStateStore();
    String appAttemptIdStr3 = "appattempt_1352994193343_0001_000003";
    ApplicationAttemptId attemptId3 =
            ConverterUtils.toApplicationAttemptId(appAttemptIdStr3);
    Path appDir =
            fsTester.store.getAppDir(attemptId3.getApplicationId().toString());
    Path tempAppAttemptFile =
            new Path(appDir, attemptId3.toString() + ".tmp");
    fsOut = fileSystemRMStateStore.fs.create(tempAppAttemptFile, false);
    fsOut.write("Some random data ".getBytes());
    fsOut.close();

    testRMAppStateStore(fsTester);
    Assert.assertFalse(fsTester.workingDirPathURI
            .getFileSystem(conf).exists(tempAppAttemptFile));
    testRMDTSecretManagerStateStore(fsTester);
    testCheckVersion(fsTester);
    testEpoch(fsTester);
    testAppDeletion(fsTester);
    testDeleteStore(fsTester);
    testAMRMTokenSecretManagerStateStore(fsTester);
  } finally {
    cluster.shutdown();
  }
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
@Test
public void testMRAppMasterMidLock() throws IOException,
    InterruptedException {
  String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
  String containerIdStr = "container_1317529182569_0004_000002_1";
  String userName = "TestAppMasterUser";
  JobConf conf = new JobConf();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  ApplicationAttemptId applicationAttemptId = ConverterUtils
      .toApplicationAttemptId(applicationAttemptIdStr);
  JobId jobId =  TypeConverter.toYarn(
      TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
  Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
  FileSystem fs = FileSystem.get(conf);
  //Create the file, but no end file so we should unregister with an error.
  fs.create(start).close();
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  MRAppMaster appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
          System.currentTimeMillis(), false, false);
  boolean caught = false;
  try {
    MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
  } catch (IOException e) {
    //The IO Exception is expected
    LOG.info("Caught expected Exception", e);
    caught = true;
  }
  assertTrue(caught);
  assertTrue(appMaster.errorHappenedShutDown);
  assertEquals(JobStateInternal.ERROR, appMaster.forcedState);
  appMaster.stop();

  // verify the final status is FAILED
  verifyFailedStatus((MRAppMasterTest)appMaster, "FAILED");
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
@Test
public void testMRAppMasterSuccessLock() throws IOException,
    InterruptedException {
  String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
  String containerIdStr = "container_1317529182569_0004_000002_1";
  String userName = "TestAppMasterUser";
  JobConf conf = new JobConf();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  ApplicationAttemptId applicationAttemptId = ConverterUtils
      .toApplicationAttemptId(applicationAttemptIdStr);
  JobId jobId =  TypeConverter.toYarn(
      TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
  Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
  Path end = MRApps.getEndJobCommitSuccessFile(conf, userName, jobId);
  FileSystem fs = FileSystem.get(conf);
  fs.create(start).close();
  fs.create(end).close();
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  MRAppMaster appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
          System.currentTimeMillis(), false, false);
  boolean caught = false;
  try {
    MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
  } catch (IOException e) {
    //The IO Exception is expected
    LOG.info("Caught expected Exception", e);
    caught = true;
  }
  assertTrue(caught);
  assertTrue(appMaster.errorHappenedShutDown);
  assertEquals(JobStateInternal.SUCCEEDED, appMaster.forcedState);
  appMaster.stop();

  // verify the final status is SUCCEEDED
  verifyFailedStatus((MRAppMasterTest)appMaster, "SUCCEEDED");
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
@Test
public void testMRAppMasterFailLock() throws IOException,
    InterruptedException {
  String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
  String containerIdStr = "container_1317529182569_0004_000002_1";
  String userName = "TestAppMasterUser";
  JobConf conf = new JobConf();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  ApplicationAttemptId applicationAttemptId = ConverterUtils
      .toApplicationAttemptId(applicationAttemptIdStr);
  JobId jobId =  TypeConverter.toYarn(
      TypeConverter.fromYarn(applicationAttemptId.getApplicationId()));
  Path start = MRApps.getStartJobCommitFile(conf, userName, jobId);
  Path end = MRApps.getEndJobCommitFailureFile(conf, userName, jobId);
  FileSystem fs = FileSystem.get(conf);
  fs.create(start).close();
  fs.create(end).close();
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  MRAppMaster appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
          System.currentTimeMillis(), false, false);
  boolean caught = false;
  try {
    MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
  } catch (IOException e) {
    //The IO Exception is expected
    LOG.info("Caught expected Exception", e);
    caught = true;
  }
  assertTrue(caught);
  assertTrue(appMaster.errorHappenedShutDown);
  assertEquals(JobStateInternal.FAILED, appMaster.forcedState);
  appMaster.stop();

  // verify the final status is FAILED
  verifyFailedStatus((MRAppMasterTest)appMaster, "FAILED");
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
@Test
public void testMRAppMasterMissingStaging() throws IOException,
    InterruptedException {
  String applicationAttemptIdStr = "appattempt_1317529182569_0004_000002";
  String containerIdStr = "container_1317529182569_0004_000002_1";
  String userName = "TestAppMasterUser";
  JobConf conf = new JobConf();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  ApplicationAttemptId applicationAttemptId = ConverterUtils
      .toApplicationAttemptId(applicationAttemptIdStr);

  //Delete the staging directory
  File dir = new File(stagingDir);
  if(dir.exists()) {
    FileUtils.deleteDirectory(dir);
  }
  
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  MRAppMaster appMaster =
      new MRAppMasterTest(applicationAttemptId, containerId, "host", -1, -1,
          System.currentTimeMillis(), false, false);
  boolean caught = false;
  try {
    MRAppMaster.initAndStartAppMaster(appMaster, conf, userName);
  } catch (IOException e) {
    //The IO Exception is expected
    LOG.info("Caught expected Exception", e);
    caught = true;
  }
  assertTrue(caught);
  assertTrue(appMaster.errorHappenedShutDown);
  //Copying the history file is disabled, but it is not really visible from 
  //here
  assertEquals(JobStateInternal.ERROR, appMaster.forcedState);
  appMaster.stop();
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
@Override
public Map<ApplicationAttemptId, ApplicationAttemptHistoryData>
    getApplicationAttempts(ApplicationId appId) throws IOException {
  Map<ApplicationAttemptId, ApplicationAttemptHistoryData> historyDataMap =
      new HashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>();
  HistoryFileReader hfReader = getHistoryFileReader(appId);
  try {
    while (hfReader.hasNext()) {
      HistoryFileReader.Entry entry = hfReader.next();
      if (entry.key.id.startsWith(
          ConverterUtils.APPLICATION_ATTEMPT_PREFIX)) {
        ApplicationAttemptId appAttemptId = 
            ConverterUtils.toApplicationAttemptId(entry.key.id);
        if (appAttemptId.getApplicationId().equals(appId)) {
          ApplicationAttemptHistoryData historyData = 
              historyDataMap.get(appAttemptId);
          if (historyData == null) {
            historyData = ApplicationAttemptHistoryData.newInstance(
                appAttemptId, null, -1, null, null, null,
                FinalApplicationStatus.UNDEFINED, null);
            historyDataMap.put(appAttemptId, historyData);
          }
          if (entry.key.suffix.equals(START_DATA_SUFFIX)) {
            mergeApplicationAttemptHistoryData(historyData,
                parseApplicationAttemptStartData(entry.value));
          } else if (entry.key.suffix.equals(FINISH_DATA_SUFFIX)) {
            mergeApplicationAttemptHistoryData(historyData,
                parseApplicationAttemptFinishData(entry.value));
          }
        }
      }
    }
    LOG.info("Completed reading history information of all application"
        + " attempts of application " + appId);
  } catch (IOException e) {
    LOG.info("Error when reading history information of some application"
        + " attempts of application " + appId);
  } finally {
    hfReader.close();
  }
  return historyDataMap;
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
@Test
public void testCommitWindow() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();

  TestingJobEventHandler jeh = new TestingJobEventHandler();
  dispatcher.register(JobEventType.class, jeh);

  SystemClock clock = new SystemClock();
  AppContext appContext = mock(AppContext.class);
  ApplicationAttemptId attemptid = 
    ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
  when(appContext.getApplicationID()).thenReturn(attemptid.getApplicationId());
  when(appContext.getApplicationAttemptId()).thenReturn(attemptid);
  when(appContext.getEventHandler()).thenReturn(
      dispatcher.getEventHandler());
  when(appContext.getClock()).thenReturn(clock);
  OutputCommitter committer = mock(OutputCommitter.class);
  TestingRMHeartbeatHandler rmhh =
      new TestingRMHeartbeatHandler();

  CommitterEventHandler ceh = new CommitterEventHandler(appContext,
      committer, rmhh);
  ceh.init(conf);
  ceh.start();

  // verify trying to commit when RM heartbeats are stale does not commit
  ceh.handle(new CommitterJobCommitEvent(null, null));
  long timeToWaitMs = 5000;
  while (rmhh.getNumCallbacks() != 1 && timeToWaitMs > 0) {
    Thread.sleep(10);
    timeToWaitMs -= 10;
  }
  Assert.assertEquals("committer did not register a heartbeat callback",
      1, rmhh.getNumCallbacks());
  verify(committer, never()).commitJob(any(JobContext.class));
  Assert.assertEquals("committer should not have committed",
      0, jeh.numCommitCompletedEvents);

  // set a fresh heartbeat and verify commit completes
  rmhh.setLastHeartbeatTime(clock.getTime());
  timeToWaitMs = 5000;
  while (jeh.numCommitCompletedEvents != 1 && timeToWaitMs > 0) {
    Thread.sleep(10);
    timeToWaitMs -= 10;
  }
  Assert.assertEquals("committer did not complete commit after RM hearbeat",
      1, jeh.numCommitCompletedEvents);
  verify(committer, times(1)).commitJob(any(JobContext.class));

  //Clean up so we can try to commit again (Don't do this at home)
  cleanup();
  
  // try to commit again and verify it goes through since the heartbeat
  // is still fresh
  ceh.handle(new CommitterJobCommitEvent(null, null));
  timeToWaitMs = 5000;
  while (jeh.numCommitCompletedEvents != 2 && timeToWaitMs > 0) {
    Thread.sleep(10);
    timeToWaitMs -= 10;
  }
  Assert.assertEquals("committer did not commit",
      2, jeh.numCommitCompletedEvents);
  verify(committer, times(2)).commitJob(any(JobContext.class));

  ceh.stop();
  dispatcher.stop();
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
@Test
public void testBasic() throws Exception {
  AppContext mockContext = mock(AppContext.class);
  OutputCommitter mockCommitter = mock(OutputCommitter.class);
  Clock mockClock = mock(Clock.class);
  
  CommitterEventHandler handler = new CommitterEventHandler(mockContext, 
      mockCommitter, new TestingRMHeartbeatHandler());
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  JobContext mockJobContext = mock(JobContext.class);
  ApplicationAttemptId attemptid = 
    ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
  JobId jobId =  TypeConverter.toYarn(
      TypeConverter.fromYarn(attemptid.getApplicationId()));
  
  WaitForItHandler waitForItHandler = new WaitForItHandler();
  
  when(mockContext.getApplicationID()).thenReturn(attemptid.getApplicationId());
  when(mockContext.getApplicationAttemptId()).thenReturn(attemptid);
  when(mockContext.getEventHandler()).thenReturn(waitForItHandler);
  when(mockContext.getClock()).thenReturn(mockClock);
  
  handler.init(conf);
  handler.start();
  try {
    handler.handle(new CommitterJobCommitEvent(jobId, mockJobContext));

    String user = UserGroupInformation.getCurrentUser().getShortUserName();
    Path startCommitFile = MRApps.getStartJobCommitFile(conf, user, jobId);
    Path endCommitSuccessFile = MRApps.getEndJobCommitSuccessFile(conf, user, 
        jobId);
    Path endCommitFailureFile = MRApps.getEndJobCommitFailureFile(conf, user, 
        jobId);

    Event e = waitForItHandler.getAndClearEvent();
    assertNotNull(e);
    assertTrue(e instanceof JobCommitCompletedEvent);
    FileSystem fs = FileSystem.get(conf);
    assertTrue(startCommitFile.toString(), fs.exists(startCommitFile));
    assertTrue(endCommitSuccessFile.toString(), fs.exists(endCommitSuccessFile));
    assertFalse(endCommitFailureFile.toString(), fs.exists(endCommitFailureFile));
    verify(mockCommitter).commitJob(any(JobContext.class));
  } finally {
    handler.stop();
  }
}
 

import org.apache.hadoop.yarn.util.ConverterUtils; //导入方法依赖的package包/类
@Test
public void testFailure() throws Exception {
  AppContext mockContext = mock(AppContext.class);
  OutputCommitter mockCommitter = mock(OutputCommitter.class);
  Clock mockClock = mock(Clock.class);
  
  CommitterEventHandler handler = new CommitterEventHandler(mockContext, 
      mockCommitter, new TestingRMHeartbeatHandler());
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  JobContext mockJobContext = mock(JobContext.class);
  ApplicationAttemptId attemptid = 
    ConverterUtils.toApplicationAttemptId("appattempt_1234567890000_0001_0");
  JobId jobId =  TypeConverter.toYarn(
      TypeConverter.fromYarn(attemptid.getApplicationId()));
  
  WaitForItHandler waitForItHandler = new WaitForItHandler();
  
  when(mockContext.getApplicationID()).thenReturn(attemptid.getApplicationId());
  when(mockContext.getApplicationAttemptId()).thenReturn(attemptid);
  when(mockContext.getEventHandler()).thenReturn(waitForItHandler);
  when(mockContext.getClock()).thenReturn(mockClock);
  
  doThrow(new YarnRuntimeException("Intentional Failure")).when(mockCommitter)
    .commitJob(any(JobContext.class));
  
  handler.init(conf);
  handler.start();
  try {
    handler.handle(new CommitterJobCommitEvent(jobId, mockJobContext));

    String user = UserGroupInformation.getCurrentUser().getShortUserName();
    Path startCommitFile = MRApps.getStartJobCommitFile(conf, user, jobId);
    Path endCommitSuccessFile = MRApps.getEndJobCommitSuccessFile(conf, user, 
        jobId);
    Path endCommitFailureFile = MRApps.getEndJobCommitFailureFile(conf, user, 
        jobId);

    Event e = waitForItHandler.getAndClearEvent();
    assertNotNull(e);
    assertTrue(e instanceof JobCommitFailedEvent);
    FileSystem fs = FileSystem.get(conf);
    assertTrue(fs.exists(startCommitFile));
    assertFalse(fs.exists(endCommitSuccessFile));
    assertTrue(fs.exists(endCommitFailureFile));
    verify(mockCommitter).commitJob(any(JobContext.class));
  } finally {
    handler.stop();
  }
}