Java RMAppAttempt.getAppAttemptId方法代码示例

import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; //导入方法依赖的package包/类
@Test
public void testAuthorizedAccess() throws Exception {
  MyContainerManager containerManager = new MyContainerManager();
  rm =
      new MockRMWithAMS(conf, containerManager);
  rm.start();

  MockNM nm1 = rm.registerNode("localhost:1234", 5120);

  Map<ApplicationAccessType, String> acls =
      new HashMap<ApplicationAccessType, String>(2);
  acls.put(ApplicationAccessType.VIEW_APP, "*");
  RMApp app = rm.submitApp(1024, "appname", "appuser", acls);

  nm1.nodeHeartbeat(true);

  int waitCount = 0;
  while (containerManager.containerTokens == null && waitCount++ < 20) {
    LOG.info("Waiting for AM Launch to happen..");
    Thread.sleep(1000);
  }
  Assert.assertNotNull(containerManager.containerTokens);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
  waitForLaunchedState(attempt);

  // Create a client to the RM.
  final Configuration conf = rm.getConfig();
  final YarnRPC rpc = YarnRPC.create(conf);

  UserGroupInformation currentUser = UserGroupInformation
      .createRemoteUser(applicationAttemptId.toString());
  Credentials credentials = containerManager.getContainerCredentials();
  final InetSocketAddress rmBindAddress =
      rm.getApplicationMasterService().getBindAddress();
  Token<? extends TokenIdentifier> amRMToken =
      MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress,
        credentials.getAllTokens());
  currentUser.addToken(amRMToken);
  ApplicationMasterProtocol client = currentUser
      .doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
          return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, rm
            .getApplicationMasterService().getBindAddress(), conf);
        }
      });

  RegisterApplicationMasterRequest request = Records
      .newRecord(RegisterApplicationMasterRequest.class);
  RegisterApplicationMasterResponse response =
      client.registerApplicationMaster(request);
  Assert.assertNotNull(response.getClientToAMTokenMasterKey());
  if (UserGroupInformation.isSecurityEnabled()) {
    Assert
      .assertTrue(response.getClientToAMTokenMasterKey().array().length > 0);
  }
  Assert.assertEquals("Register response has bad ACLs", "*",
      response.getApplicationACLs().get(ApplicationAccessType.VIEW_APP));
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; //导入方法依赖的package包/类
@Test (timeout = 300000)
public void testActivatingApplicationAfterAddingNM() throws Exception {
  MockRM rm1 = new MockRM(conf);

  // start like normal because state is empty
  rm1.start();

  // app that gets launched
  RMApp app1 = rm1.submitApp(200);

  // app that does not get launched
  RMApp app2 = rm1.submitApp(200);

  // app1 and app2 should be scheduled, but because no resource is available,
  // they are not activated.
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId();
  rm1.waitForState(attemptId1, RMAppAttemptState.SCHEDULED);
  RMAppAttempt attempt2 = app2.getCurrentAppAttempt();
  ApplicationAttemptId attemptId2 = attempt2.getAppAttemptId();
  rm1.waitForState(attemptId2, RMAppAttemptState.SCHEDULED);

  MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
  MockNM nm2 = new MockNM("h2:5678", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();
  nm2.registerNode();

  //kick the scheduling
  nm1.nodeHeartbeat(true);

  // app1 should be allocated now
  rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
  rm1.waitForState(attemptId2, RMAppAttemptState.SCHEDULED);

  nm2.nodeHeartbeat(true);

  // app2 should be allocated now
  rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
  rm1.waitForState(attemptId2, RMAppAttemptState.ALLOCATED);

  rm1.stop();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; //导入方法依赖的package包/类
@Test (timeout = 60000)
public void testRMRestartOnMaxAppAttempts() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
      YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);

  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  RMState rmState = memStore.getState();

  Map<ApplicationId, ApplicationStateData> rmAppState =
      rmState.getApplicationState();  
  MockRM rm1 = createMockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  // submit an app with maxAppAttempts equals to 1
  RMApp app1 = rm1.submitApp(200, "name", "user",
        new HashMap<ApplicationAccessType, String>(), false, "default", 1,
        null);
  // submit an app with maxAppAttempts equals to -1
  RMApp app2 = rm1.submitApp(200, "name", "user",
        new HashMap<ApplicationAccessType, String>(), false, "default", -1,
        null);

  // assert app1 info is saved
  ApplicationStateData appState = rmAppState.get(app1.getApplicationId());
  Assert.assertNotNull(appState);
  Assert.assertEquals(0, appState.getAttemptCount());
  Assert.assertEquals(appState.getApplicationSubmissionContext()
      .getApplicationId(), app1.getApplicationSubmissionContext()
      .getApplicationId());

  // Allocate the AM
  nm1.nodeHeartbeat(true);
  RMAppAttempt attempt = app1.getCurrentAppAttempt();
  ApplicationAttemptId attemptId1 = attempt.getAppAttemptId();
  rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
  Assert.assertEquals(1, appState.getAttemptCount());
  ApplicationAttemptStateData attemptState =
                              appState.getAttempt(attemptId1);
  Assert.assertNotNull(attemptState);
  Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1), 
                      attemptState.getMasterContainer().getId());

  // Setting AMLivelinessMonitor interval to be 3 Secs.
  conf.setInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 3000);
  // start new RM   
  MockRM rm2 = createMockRM(conf, memStore);
  rm2.start();

  // verify that maxAppAttempts is set to global value
  Assert.assertEquals(2, 
      rm2.getRMContext().getRMApps().get(app2.getApplicationId())
      .getMaxAppAttempts());

  // app1 and app2 are loaded back, but app1 failed because it's
  // hitting max-retry.
  Assert.assertEquals(2, rm2.getRMContext().getRMApps().size());
  rm2.waitForState(app1.getApplicationId(), RMAppState.FAILED);
  rm2.waitForState(app2.getApplicationId(), RMAppState.ACCEPTED);

  // app1 failed state is saved in state store. app2 final saved state is not
  // determined yet.
  Assert.assertEquals(RMAppState.FAILED,
    rmAppState.get(app1.getApplicationId()).getState());
  Assert.assertNull(rmAppState.get(app2.getApplicationId()).getState());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; //导入方法依赖的package包/类
@Test (timeout = 60000)
public void testAppAttemptTokensRestoredOnRMRestart() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    "kerberos");
  UserGroupInformation.setConfiguration(conf);

  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  RMState rmState = memStore.getState();

  Map<ApplicationId, ApplicationStateData> rmAppState =
      rmState.getApplicationState();
  MockRM rm1 = new TestSecurityMockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("0.0.0.0:4321", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  // submit an app
  RMApp app1 =
      rm1.submitApp(200, "name", "user",
        new HashMap<ApplicationAccessType, String>(), "default");

  // assert app info is saved
  ApplicationStateData appState = rmAppState.get(app1.getApplicationId());
  Assert.assertNotNull(appState);

  // Allocate the AM
  nm1.nodeHeartbeat(true);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId();
  rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);

  // assert attempt info is saved
  ApplicationAttemptStateData attemptState = appState.getAttempt(attemptId1);
  Assert.assertNotNull(attemptState);
  Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1),
    attemptState.getMasterContainer().getId());

  // the clientTokenMasterKey that are generated when
  // RMAppAttempt is created,
  byte[] clientTokenMasterKey =
      attempt1.getClientTokenMasterKey().getEncoded();

  // assert application credentials are saved
  Credentials savedCredentials = attemptState.getAppAttemptTokens();
  Assert.assertArrayEquals("client token master key not saved",
      clientTokenMasterKey, savedCredentials.getSecretKey(
          RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));

  // start new RM
  MockRM rm2 = new TestSecurityMockRM(conf, memStore);
  rm2.start();

  RMApp loadedApp1 =
      rm2.getRMContext().getRMApps().get(app1.getApplicationId());
  RMAppAttempt loadedAttempt1 = loadedApp1.getRMAppAttempt(attemptId1);

  // assert loaded attempt recovered
  Assert.assertNotNull(loadedAttempt1);

  // assert client token master key is recovered back to api-versioned
  // client token master key
  Assert.assertEquals("client token master key not restored",
      attempt1.getClientTokenMasterKey(),
      loadedAttempt1.getClientTokenMasterKey());

  // assert ClientTokenSecretManager also knows about the key
  Assert.assertArrayEquals(clientTokenMasterKey,
      rm2.getClientToAMTokenSecretManager().getMasterKey(attemptId1)
          .getEncoded());

  // assert AMRMTokenSecretManager also knows about the AMRMToken password
  Token<AMRMTokenIdentifier> amrmToken = loadedAttempt1.getAMRMToken();
  Assert.assertArrayEquals(amrmToken.getPassword(),
    rm2.getRMContext().getAMRMTokenSecretManager().retrievePassword(
      amrmToken.decodeIdentifier()));
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; //导入方法依赖的package包/类
@Test
public void testUnauthorizedAccess() throws Exception {
  MyContainerManager containerManager = new MyContainerManager();
  rm = new MockRMWithAMS(conf, containerManager);
  rm.start();

  MockNM nm1 = rm.registerNode("localhost:1234", 5120);

  RMApp app = rm.submitApp(1024);

  nm1.nodeHeartbeat(true);

  int waitCount = 0;
  while (containerManager.containerTokens == null && waitCount++ < 40) {
    LOG.info("Waiting for AM Launch to happen..");
    Thread.sleep(1000);
  }
  Assert.assertNotNull(containerManager.containerTokens);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
  waitForLaunchedState(attempt);

  final Configuration conf = rm.getConfig();
  final YarnRPC rpc = YarnRPC.create(conf);
  final InetSocketAddress serviceAddr = conf.getSocketAddr(
      YarnConfiguration.RM_SCHEDULER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);

  UserGroupInformation currentUser = UserGroupInformation
      .createRemoteUser(applicationAttemptId.toString());

  // First try contacting NM without tokens
  ApplicationMasterProtocol client = currentUser
      .doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
          return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class,
              serviceAddr, conf);
        }
      });
  
  RegisterApplicationMasterRequest request = Records
      .newRecord(RegisterApplicationMasterRequest.class);
  try {
    client.registerApplicationMaster(request);
    Assert.fail("Should fail with authorization error");
  } catch (Exception e) {
    if (isCause(AccessControlException.class, e)) {
      // Because there are no tokens, the request should be rejected as the
      // server side will assume we are trying simple auth.
      String expectedMessage = "";
      if (UserGroupInformation.isSecurityEnabled()) {
        expectedMessage = "Client cannot authenticate via:[TOKEN]";
      } else {
        expectedMessage =
            "SIMPLE authentication is not enabled.  Available:[TOKEN]";
      }
      Assert.assertTrue(e.getCause().getMessage().contains(expectedMessage)); 
    } else {
      throw e;
    }
  }

  // TODO: Add validation of invalid authorization when there's more data in
  // the AMRMToken
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; //导入方法依赖的package包/类
@Test
public void testValidateResourceBlacklistRequest() throws Exception {

  MyContainerManager containerManager = new MyContainerManager();
  final MockRMWithAMS rm =
      new MockRMWithAMS(new YarnConfiguration(), containerManager);
  rm.start();

  MockNM nm1 = rm.registerNode("localhost:1234", 5120);

  Map<ApplicationAccessType, String> acls =
      new HashMap<ApplicationAccessType, String>(2);
  acls.put(ApplicationAccessType.VIEW_APP, "*");
  RMApp app = rm.submitApp(1024, "appname", "appuser", acls);

  nm1.nodeHeartbeat(true);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
  waitForLaunchedState(attempt);

  // Create a client to the RM.
  final Configuration conf = rm.getConfig();
  final YarnRPC rpc = YarnRPC.create(conf);

  UserGroupInformation currentUser = 
      UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
  Credentials credentials = containerManager.getContainerCredentials();
  final InetSocketAddress rmBindAddress =
      rm.getApplicationMasterService().getBindAddress();
  Token<? extends TokenIdentifier> amRMToken =
      MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress,
        credentials.getAllTokens());
  currentUser.addToken(amRMToken);
  ApplicationMasterProtocol client =
      currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
          return (ApplicationMasterProtocol) rpc.getProxy(
            ApplicationMasterProtocol.class, rmBindAddress, conf);
        }
      });

  RegisterApplicationMasterRequest request = Records
      .newRecord(RegisterApplicationMasterRequest.class);
  client.registerApplicationMaster(request);

  ResourceBlacklistRequest blacklistRequest =
      ResourceBlacklistRequest.newInstance(
          Collections.singletonList(ResourceRequest.ANY), null);

  AllocateRequest allocateRequest =
      AllocateRequest.newInstance(0, 0.0f, null, null, blacklistRequest);
  boolean error = false;
  try {
    client.allocate(allocateRequest);
  } catch (InvalidResourceBlacklistRequestException e) {
    error = true;
  }

  rm.stop();
  
  Assert.assertTrue(
      "Didn't not catch InvalidResourceBlacklistRequestException", error);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; //导入方法依赖的package包/类
@Test
public void testAMLaunchAndCleanup() throws Exception {
  Logger rootLogger = LogManager.getRootLogger();
  rootLogger.setLevel(Level.DEBUG);
  MyContainerManagerImpl containerManager = new MyContainerManagerImpl();
  MockRMWithCustomAMLauncher rm = new MockRMWithCustomAMLauncher(
      containerManager);
  rm.start();
  MockNM nm1 = rm.registerNode("127.0.0.1:1234", 5120);

  RMApp app = rm.submitApp(2000);

  // kick the scheduling
  nm1.nodeHeartbeat(true);

  int waitCount = 0;
  while (containerManager.launched == false && waitCount++ < 20) {
    LOG.info("Waiting for AM Launch to happen..");
    Thread.sleep(1000);
  }
  Assert.assertTrue(containerManager.launched);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  ApplicationAttemptId appAttemptId = attempt.getAppAttemptId();
  Assert.assertEquals(appAttemptId.toString(),
      containerManager.attemptIdAtContainerManager);
  Assert.assertEquals(app.getSubmitTime(),
      containerManager.submitTimeAtContainerManager);
  Assert.assertEquals(app.getRMAppAttempt(appAttemptId)
      .getMasterContainer().getId()
      .toString(), containerManager.containerIdAtContainerManager);
  Assert.assertEquals(nm1.getNodeId().toString(),
    containerManager.nmHostAtContainerManager);
  Assert.assertEquals(YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS,
      containerManager.maxAppAttempts);

  MockAM am = new MockAM(rm.getRMContext(), rm
      .getApplicationMasterService(), appAttemptId);
  am.registerAppAttempt();
  am.unregisterAppAttempt();

  //complete the AM container to finish the app normally
  nm1.nodeHeartbeat(attempt.getAppAttemptId(), 1, ContainerState.COMPLETE);
  am.waitForState(RMAppAttemptState.FINISHED);

  waitCount = 0;
  while (containerManager.cleanedup == false && waitCount++ < 20) {
    LOG.info("Waiting for AM Cleanup to happen..");
    Thread.sleep(1000);
  }
  Assert.assertTrue(containerManager.cleanedup);

  am.waitForState(RMAppAttemptState.FINISHED);
  rm.stop();
}