Java MockRM.getResourceTrackerService方法代码示例

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test(timeout = 10000)
public void testRMNodeStatusAfterReconnect() throws Exception {
  // The node(127.0.0.1:1234) reconnected with RM. When it registered with
  // RM, RM set its lastNodeHeartbeatResponse's id to 0 asynchronously. But
  // the node's heartbeat come before RM succeeded setting the id to 0.
  final DrainDispatcher dispatcher = new DrainDispatcher();
  MockRM rm = new MockRM(){
    @Override
    protected Dispatcher createDispatcher() {
      return dispatcher;
    }
  };
  rm.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
  nm1.registerNode();
  int i = 0;
  while(i < 3) {
    nm1.nodeHeartbeat(true);
    dispatcher.await();
    i++;
  }

  MockNM nm2 =
      new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
  nm2.registerNode();
  RMNode rmNode = rm.getRMContext().getRMNodes().get(nm2.getNodeId());
  nm2.nodeHeartbeat(true);
  dispatcher.await();
  Assert.assertEquals("Node is Not in Running state.", NodeState.RUNNING,
      rmNode.getState());
  rm.stop();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test
public void testAppSubmissionWithoutDelegationToken() throws Exception {
  conf.setBoolean(YarnConfiguration.RM_PROXY_USER_PRIVILEGES_ENABLED, true);
  // create token2
  Text userText2 = new Text("user2");
  DelegationTokenIdentifier dtId2 =
      new DelegationTokenIdentifier(new Text("user2"), new Text("renewer2"),
        userText2);
  final Token<DelegationTokenIdentifier> token2 =
      new Token<DelegationTokenIdentifier>(dtId2.getBytes(),
        "password2".getBytes(), dtId2.getKind(), new Text("service2"));
  final MockRM rm = new TestSecurityMockRM(conf, null) {
    @Override
    protected DelegationTokenRenewer createDelegationTokenRenewer() {
      return new DelegationTokenRenewer() {
        @Override
        protected Token<?>[] obtainSystemTokensForUser(String user,
            final Credentials credentials) throws IOException {
          credentials.addToken(token2.getService(), token2);
          return new Token<?>[] { token2 };
        }
      };
    }
  };
  rm.start();

  // submit an app without delegationToken
  RMApp app = rm.submitApp(200);

  // wait for the new retrieved hdfs token.
  GenericTestUtils.waitFor(new Supplier<Boolean>() {
    public Boolean get() {
      return rm.getRMContext().getDelegationTokenRenewer()
        .getDelegationTokens().contains(token2);
    }
  }, 1000, 20000);

  // check nm can retrieve the token
  final MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
  nm1.registerNode();
  NodeHeartbeatResponse response = nm1.nodeHeartbeat(true);
  ByteBuffer tokenBuffer =
      response.getSystemCredentialsForApps().get(app.getApplicationId());
  Assert.assertNotNull(tokenBuffer);
  Credentials appCredentials = new Credentials();
  DataInputByteBuffer buf = new DataInputByteBuffer();
  tokenBuffer.rewind();
  buf.reset(tokenBuffer);
  appCredentials.readTokenStorageStream(buf);
  Assert.assertTrue(appCredentials.getAllTokens().contains(token2));
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test (timeout = 30000)
public void testAppSubmissionWithPreviousToken() throws Exception{
  MockRM rm = new TestSecurityMockRM(conf, null);
  rm.start();
  final MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
  nm1.registerNode();

  // create Token1:
  Text userText1 = new Text("user");
  DelegationTokenIdentifier dtId1 =
      new DelegationTokenIdentifier(userText1, new Text("renewer1"),
        userText1);
  final Token<DelegationTokenIdentifier> token1 =
      new Token<DelegationTokenIdentifier>(dtId1.getBytes(),
        "password1".getBytes(), dtId1.getKind(), new Text("service1"));

  Credentials credentials = new Credentials();
  credentials.addToken(userText1, token1);

  // submit app1 with a token, set cancelTokenWhenComplete to false;
  RMApp app1 =
      rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
        null, true, false, false, null, 0, null, false);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
  rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING);

  // submit app2 with the same token, set cancelTokenWhenComplete to true;
  RMApp app2 =
      rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
        null, true, false, false, null, 0, null, true);
  MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
  rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING);
  MockRM.finishAMAndVerifyAppState(app2, rm, nm1, am2);
  Assert.assertTrue(rm.getRMContext().getDelegationTokenRenewer()
    .getAllTokens().containsKey(token1));

  MockRM.finishAMAndVerifyAppState(app1, rm, nm1, am1);
  // app2 completes, app1 is still running, check the token is not cancelled
  Assert.assertFalse(Renewer.cancelled);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test (timeout = 30000)
public void testCancelWithMultipleAppSubmissions() throws Exception{
  MockRM rm = new TestSecurityMockRM(conf, null);
  rm.start();
  final MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
  nm1.registerNode();

  //MyFS fs = (MyFS)FileSystem.get(conf);
  //MyToken token1 = fs.getDelegationToken("user123");

  // create Token1:
  Text userText1 = new Text("user");
  DelegationTokenIdentifier dtId1 =
      new DelegationTokenIdentifier(userText1, new Text("renewer1"),
        userText1);
  final Token<DelegationTokenIdentifier> token1 =
      new Token<DelegationTokenIdentifier>(dtId1.getBytes(),
        "password1".getBytes(), dtId1.getKind(), new Text("service1"));

  Credentials credentials = new Credentials();
  credentials.addToken(token1.getService(), token1);

  DelegationTokenRenewer renewer =
      rm.getRMContext().getDelegationTokenRenewer();
  Assert.assertTrue(renewer.getAllTokens().isEmpty());
  Assert.assertFalse(Renewer.cancelled);

  RMApp app1 =
      rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
        null, true, false, false, null, 0, null, true);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
  rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING);

  DelegationTokenToRenew dttr = renewer.getAllTokens().get(token1);
  Assert.assertNotNull(dttr);
  Assert.assertTrue(dttr.referringAppIds.contains(app1.getApplicationId()));
  RMApp app2 =
      rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
        null, true, false, false, null, 0, null, true);
  MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
  rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING);
  Assert.assertTrue(renewer.getAllTokens().containsKey(token1));
  Assert.assertTrue(dttr.referringAppIds.contains(app2.getApplicationId()));
  Assert.assertTrue(dttr.referringAppIds.contains(app2.getApplicationId()));
  Assert.assertFalse(Renewer.cancelled);

  MockRM.finishAMAndVerifyAppState(app2, rm, nm1, am2);
  // app2 completes, app1 is still running, check the token is not cancelled
  Assert.assertTrue(renewer.getAllTokens().containsKey(token1));
  Assert.assertTrue(dttr.referringAppIds.contains(app1.getApplicationId()));
  Assert.assertFalse(dttr.referringAppIds.contains(app2.getApplicationId()));
  Assert.assertFalse(dttr.isTimerCancelled());
  Assert.assertFalse(Renewer.cancelled);

  RMApp app3 =
      rm.submitApp(200, "name", "user", null, false, null, 2, credentials,
        null, true, false, false, null, 0, null, true);
  MockAM am3 = MockRM.launchAndRegisterAM(app3, rm, nm1);
  rm.waitForState(app3.getApplicationId(), RMAppState.RUNNING);
  Assert.assertTrue(renewer.getAllTokens().containsKey(token1));
  Assert.assertTrue(dttr.referringAppIds.contains(app1.getApplicationId()));
  Assert.assertTrue(dttr.referringAppIds.contains(app3.getApplicationId()));
  Assert.assertFalse(dttr.isTimerCancelled());
  Assert.assertFalse(Renewer.cancelled);

  MockRM.finishAMAndVerifyAppState(app1, rm, nm1, am1);
  Assert.assertTrue(renewer.getAllTokens().containsKey(token1));
  Assert.assertFalse(dttr.referringAppIds.contains(app1.getApplicationId()));
  Assert.assertTrue(dttr.referringAppIds.contains(app3.getApplicationId()));
  Assert.assertFalse(dttr.isTimerCancelled());
  Assert.assertFalse(Renewer.cancelled);

  MockRM.finishAMAndVerifyAppState(app3, rm, nm1, am3);
  Assert.assertFalse(renewer.getAllTokens().containsKey(token1));
  Assert.assertTrue(dttr.referringAppIds.isEmpty());
  Assert.assertTrue(dttr.isTimerCancelled());
  Assert.assertTrue(Renewer.cancelled);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test(timeout = 20000)
public void testPreemptedAMRestartOnRMRestart() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
  conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);

  conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
  // explicitly set max-am-retry count as 1.
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  MockRM rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1 = rm1.submitApp(200);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  CapacityScheduler scheduler =
      (CapacityScheduler) rm1.getResourceScheduler();
  ContainerId amContainer =
      ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);

  // Forcibly preempt the am container;
  scheduler.killContainer(scheduler.getRMContainer(amContainer));

  am1.waitForState(RMAppAttemptState.FAILED);
  Assert.assertTrue(! attempt1.shouldCountTowardsMaxAttemptRetry());
  rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);

  // state store has 1 attempt stored.
  ApplicationStateData appState =
      memStore.getState().getApplicationState().get(app1.getApplicationId());
  Assert.assertEquals(1, appState.getAttemptCount());
  // attempt stored has the preempted container exit status.
  Assert.assertEquals(ContainerExitStatus.PREEMPTED,
    appState.getAttempt(am1.getApplicationAttemptId())
      .getAMContainerExitStatus());
  // Restart rm.
  MockRM rm2 = new MockRM(conf, memStore);
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  nm1.registerNode();
  rm2.start();

  // Restarted RM should re-launch the am.
  MockAM am2 =
      rm2.waitForNewAMToLaunchAndRegister(app1.getApplicationId(), 2, nm1);
  MockRM.finishAMAndVerifyAppState(app1, rm2, nm1, am2);
  RMAppAttempt attempt2 =
      rm2.getRMContext().getRMApps().get(app1.getApplicationId())
        .getCurrentAppAttempt();
  Assert.assertTrue(attempt2.shouldCountTowardsMaxAttemptRetry());
  Assert.assertEquals(ContainerExitStatus.INVALID,
    appState.getAttempt(am2.getApplicationAttemptId())
      .getAMContainerExitStatus());
  rm1.stop();
  rm2.stop();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test(timeout = 50000)
public void testRMRestartOrFailoverNotCountedForAMFailures()
    throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
  conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);

  conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
  // explicitly set max-am-retry count as 1.
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  MockRM rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1 = rm1.submitApp(200);
  // AM should be restarted even though max-am-attempt is 1.
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  Assert.assertTrue(((RMAppAttemptImpl) attempt1).mayBeLastAttempt());

  // Restart rm.
  MockRM rm2 = new MockRM(conf, memStore);
  rm2.start();
  ApplicationStateData appState =
      memStore.getState().getApplicationState().get(app1.getApplicationId());
  // re-register the NM
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  NMContainerStatus status = Records.newRecord(NMContainerStatus.class);
  status
    .setContainerExitStatus(ContainerExitStatus.KILLED_BY_RESOURCEMANAGER);
  status.setContainerId(attempt1.getMasterContainer().getId());
  status.setContainerState(ContainerState.COMPLETE);
  status.setDiagnostics("");
  nm1.registerNode(Collections.singletonList(status), null);

  rm2.waitForState(attempt1.getAppAttemptId(), RMAppAttemptState.FAILED);
  Assert.assertEquals(ContainerExitStatus.KILLED_BY_RESOURCEMANAGER,
    appState.getAttempt(am1.getApplicationAttemptId())
      .getAMContainerExitStatus());
  // Will automatically start a new AppAttempt in rm2
  rm2.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
  MockAM am2 =
      rm2.waitForNewAMToLaunchAndRegister(app1.getApplicationId(), 2, nm1);
  MockRM.finishAMAndVerifyAppState(app1, rm2, nm1, am2);
  RMAppAttempt attempt3 =
      rm2.getRMContext().getRMApps().get(app1.getApplicationId())
        .getCurrentAppAttempt();
  Assert.assertTrue(attempt3.shouldCountTowardsMaxAttemptRetry());
  Assert.assertEquals(ContainerExitStatus.INVALID,
    appState.getAttempt(am2.getApplicationAttemptId())
      .getAMContainerExitStatus());

  rm1.stop();
  rm2.stop();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test
public void testHierarchyQueuesCurrentLimits() throws Exception {
  /*
   * Queue tree:
   *          Root
   *        /     \
   *       A       B
   *      / \    / | \
   *     A1 A2  B1 B2 B3
   */
  YarnConfiguration conf =
      new YarnConfiguration(
          setupQueueConfiguration(new CapacitySchedulerConfiguration()));
  conf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true);

  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  MockRM rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 100 * GB, rm1.getResourceTrackerService());
  nm1.registerNode();
  
  RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "b1");
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  
  waitContainerAllocated(am1, 1 * GB, 1, 2, rm1, nm1);

  // Maximum resoure of b1 is 100 * 0.895 * 0.792 = 71 GB
  // 2 GBs used by am, so it's 71 - 2 = 69G.
  Assert.assertEquals(69 * GB,
      am1.doHeartbeat().getAvailableResources().getMemory());
  
  RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "b2");
  MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
  
  // Allocate 5 containers, each one is 8 GB in am2 (40 GB in total)
  waitContainerAllocated(am2, 8 * GB, 5, 2, rm1, nm1);
  
  // Allocated one more container with 1 GB resource in b1
  waitContainerAllocated(am1, 1 * GB, 1, 3, rm1, nm1);
  
  // Total is 100 GB, 
  // B2 uses 41 GB (5 * 8GB containers and 1 AM container)
  // B1 uses 3 GB (2 * 1GB containers and 1 AM container)
  // Available is 100 - 41 - 3 = 56 GB
  Assert.assertEquals(56 * GB,
      am1.doHeartbeat().getAvailableResources().getMemory());
  
  // Now we submit app3 to a1 (in higher level hierarchy), to see if headroom
  // of app1 (in queue b1) updated correctly
  RMApp app3 = rm1.submitApp(1 * GB, "app", "user", null, "a1");
  MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1);
  
  // Allocate 3 containers, each one is 8 GB in am3 (24 GB in total)
  waitContainerAllocated(am3, 8 * GB, 3, 2, rm1, nm1);
  
  // Allocated one more container with 4 GB resource in b1
  waitContainerAllocated(am1, 1 * GB, 1, 4, rm1, nm1);
  
  // Total is 100 GB, 
  // B2 uses 41 GB (5 * 8GB containers and 1 AM container)
  // B1 uses 4 GB (3 * 1GB containers and 1 AM container)
  // A1 uses 25 GB (3 * 8GB containers and 1 AM container)
  // Available is 100 - 41 - 4 - 25 = 30 GB
  Assert.assertEquals(30 * GB,
      am1.doHeartbeat().getAvailableResources().getMemory());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test
public void testParentQueueMaxCapsAreRespected() throws Exception {
  /*
   * Queue tree:
   *          Root
   *        /     \
   *       A       B
   *      / \
   *     A1 A2 
   */
  CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
  csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a", "b"});
  csConf.setCapacity(A, 50);
  csConf.setMaximumCapacity(A, 50);
  csConf.setCapacity(B, 50);
  
  // Define 2nd-level queues
  csConf.setQueues(A, new String[] {"a1", "a2"});
  csConf.setCapacity(A1, 50);
  csConf.setUserLimitFactor(A1, 100.0f);
  csConf.setCapacity(A2, 50);
  csConf.setUserLimitFactor(A2, 100.0f);
  csConf.setCapacity(B1, B1_CAPACITY);
  csConf.setUserLimitFactor(B1, 100.0f);
  
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true);

  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  MockRM rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 24 * GB, rm1.getResourceTrackerService());
  nm1.registerNode();
  
  // Launch app1 in a1, resource usage is 1GB (am) + 4GB * 2 = 9GB 
  RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a1");
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  waitContainerAllocated(am1, 4 * GB, 2, 2, rm1, nm1);
  
  // Try to launch app2 in a2, asked 2GB, should success 
  RMApp app2 = rm1.submitApp(2 * GB, "app", "user", null, "a2");
  MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
  try {
    // Try to allocate a container, a's usage=11G/max=12
    // a1's usage=9G/max=12
    // a2's usage=2G/max=12
    // In this case, if a2 asked 2G, should fail.
    waitContainerAllocated(am2, 2 * GB, 1, 2, rm1, nm1);
  } catch (AssertionError failure) {
    // Expected, return;
    return;
  }
  Assert.fail("Shouldn't successfully allocate containers for am2, "
      + "queue-a's max capacity will be violated if container allocated");
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test (timeout = 30000)
public void testAppSubmissionWithPreviousToken() throws Exception{
  MockRM rm = new TestSecurityMockRM(conf, null);
  rm.start();
  final MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
  nm1.registerNode();

  // create Token1:
  Text userText1 = new Text("user");
  DelegationTokenIdentifier dtId1 =
      new DelegationTokenIdentifier(userText1, new Text("renewer1"),
        userText1);
  final Token<DelegationTokenIdentifier> token1 =
      new Token<DelegationTokenIdentifier>(dtId1.getBytes(),
        "password1".getBytes(), dtId1.getKind(), new Text("service1"));

  Credentials credentials = new Credentials();
  credentials.addToken(userText1, token1);

  // submit app1 with a token, set cancelTokenWhenComplete to false;
  Resource resource = Records.newRecord(Resource.class);
  resource.setMemory(200);
  RMApp app1 = rm.submitApp(resource, "name", "user", null, false, null, 2,
      credentials, null, true, false, false, null, 0, null, false, null);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
  rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING);

  // submit app2 with the same token, set cancelTokenWhenComplete to true;
  RMApp app2 = rm.submitApp(resource, "name", "user", null, false, null, 2,
      credentials, null, true, false, false, null, 0, null, true, null);
  MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
  rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING);
  MockRM.finishAMAndVerifyAppState(app2, rm, nm1, am2);
  Assert.assertTrue(rm.getRMContext().getDelegationTokenRenewer()
    .getAllTokens().containsKey(token1));

  MockRM.finishAMAndVerifyAppState(app1, rm, nm1, am1);
  // app2 completes, app1 is still running, check the token is not cancelled
  Assert.assertFalse(Renewer.cancelled);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test (timeout = 30000)
public void testCancelWithMultipleAppSubmissions() throws Exception{
  MockRM rm = new TestSecurityMockRM(conf, null);
  rm.start();
  final MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
  nm1.registerNode();

  //MyFS fs = (MyFS)FileSystem.get(conf);
  //MyToken token1 = fs.getDelegationToken("user123");

  // create Token1:
  Text userText1 = new Text("user");
  DelegationTokenIdentifier dtId1 =
      new DelegationTokenIdentifier(userText1, new Text("renewer1"),
        userText1);
  final Token<DelegationTokenIdentifier> token1 =
      new Token<DelegationTokenIdentifier>(dtId1.getBytes(),
        "password1".getBytes(), dtId1.getKind(), new Text("service1"));

  Credentials credentials = new Credentials();
  credentials.addToken(token1.getService(), token1);

  DelegationTokenRenewer renewer =
      rm.getRMContext().getDelegationTokenRenewer();
  Assert.assertTrue(renewer.getAllTokens().isEmpty());
  Assert.assertFalse(Renewer.cancelled);

  Resource resource = Records.newRecord(Resource.class);
  resource.setMemory(200);
  RMApp app1 =
      rm.submitApp(resource, "name", "user", null, false, null, 2, credentials,
        null, true, false, false, null, 0, null, true, null);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
  rm.waitForState(app1.getApplicationId(), RMAppState.RUNNING);

  DelegationTokenToRenew dttr = renewer.getAllTokens().get(token1);
  Assert.assertNotNull(dttr);
  Assert.assertTrue(dttr.referringAppIds.contains(app1.getApplicationId()));
  RMApp app2 = rm.submitApp(resource, "name", "user", null, false, null, 2,
      credentials, null, true, false, false, null, 0, null, true, null);
  MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm1);
  rm.waitForState(app2.getApplicationId(), RMAppState.RUNNING);
  Assert.assertTrue(renewer.getAllTokens().containsKey(token1));
  Assert.assertTrue(dttr.referringAppIds.contains(app2.getApplicationId()));
  Assert.assertTrue(dttr.referringAppIds.contains(app2.getApplicationId()));
  Assert.assertFalse(Renewer.cancelled);

  MockRM.finishAMAndVerifyAppState(app2, rm, nm1, am2);
  // app2 completes, app1 is still running, check the token is not cancelled
  Assert.assertTrue(renewer.getAllTokens().containsKey(token1));
  Assert.assertTrue(dttr.referringAppIds.contains(app1.getApplicationId()));
  Assert.assertFalse(dttr.referringAppIds.contains(app2.getApplicationId()));
  Assert.assertFalse(dttr.isTimerCancelled());
  Assert.assertFalse(Renewer.cancelled);

  RMApp app3 = rm.submitApp(resource, "name", "user", null, false, null, 2,
      credentials, null, true, false, false, null, 0, null, true, null);
  MockAM am3 = MockRM.launchAndRegisterAM(app3, rm, nm1);
  rm.waitForState(app3.getApplicationId(), RMAppState.RUNNING);
  Assert.assertTrue(renewer.getAllTokens().containsKey(token1));
  Assert.assertTrue(dttr.referringAppIds.contains(app1.getApplicationId()));
  Assert.assertTrue(dttr.referringAppIds.contains(app3.getApplicationId()));
  Assert.assertFalse(dttr.isTimerCancelled());
  Assert.assertFalse(Renewer.cancelled);

  MockRM.finishAMAndVerifyAppState(app1, rm, nm1, am1);
  Assert.assertTrue(renewer.getAllTokens().containsKey(token1));
  Assert.assertFalse(dttr.referringAppIds.contains(app1.getApplicationId()));
  Assert.assertTrue(dttr.referringAppIds.contains(app3.getApplicationId()));
  Assert.assertFalse(dttr.isTimerCancelled());
  Assert.assertFalse(Renewer.cancelled);

  MockRM.finishAMAndVerifyAppState(app3, rm, nm1, am3);
  Assert.assertFalse(renewer.getAllTokens().containsKey(token1));
  Assert.assertTrue(dttr.referringAppIds.isEmpty());
  Assert.assertTrue(dttr.isTimerCancelled());
  Assert.assertTrue(Renewer.cancelled);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test(timeout = 180000)
public void testRMRestartWithChangeInPriority() throws Exception {
  conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
  conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED,
      false);
  conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
      YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);

  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  RMState rmState = memStore.getState();
  Map<ApplicationId, ApplicationStateData> rmAppState = rmState
      .getApplicationState();

  // PHASE 1: create state in an RM

  // start RM
  MockRM rm1 = new MockRM(conf, memStore);
  rm1.start();

  MockNM nm1 = new MockNM("127.0.0.1:1234", 15120,
      rm1.getResourceTrackerService());
  nm1.registerNode();

  Priority appPriority1 = Priority.newInstance(5);
  RMApp app1 = rm1.submitApp(1 * GB, appPriority1);

  // kick the scheduler, 1 GB given to AM1, remaining 15GB on nm1
  MockAM am1 = MockRM.launchAM(app1, rm1, nm1);
  am1.registerAppAttempt();

  // get scheduler
  CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();

  // Change the priority of App1 to 8
  Priority appPriority2 = Priority.newInstance(8);
  cs.updateApplicationPriority(appPriority2, app1.getApplicationId());

  // let things settle down
  Thread.sleep(1000);

  // create new RM to represent restart and recover state
  MockRM rm2 = new MockRM(conf, memStore);

  // start new RM
  rm2.start();
  // change NM to point to new RM
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());

  // Verify RM Apps after this restart
  Assert.assertEquals(1, rm2.getRMContext().getRMApps().size());

  // get scheduler app
  RMApp loadedApp = rm2.getRMContext().getRMApps()
      .get(app1.getApplicationId());

  // Verify whether priority 15 is reset to 10
  Assert.assertEquals(appPriority2, loadedApp.getCurrentAppAttempt()
      .getSubmissionContext().getPriority());

  rm2.stop();
  rm1.stop();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@SuppressWarnings({ "rawtypes" })
@Test(timeout = 10000)
public void testReleasedContainerIfAppAttemptisNull() throws Exception {
  YarnConfiguration conf=getConf();
  conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  MockRM rm1 = new MockRM(conf, memStore);
  try {
    rm1.start();
    MockNM nm1 =
        new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
    nm1.registerNode();

    AbstractYarnScheduler scheduler =
        (AbstractYarnScheduler) rm1.getResourceScheduler();
    // Mock App without attempt
    RMApp mockAPp =
        new MockRMApp(125, System.currentTimeMillis(), RMAppState.NEW);
    SchedulerApplication<FiCaSchedulerApp> application =
        new SchedulerApplication<FiCaSchedulerApp>(null, mockAPp.getUser());

    // Second app with one app attempt
    RMApp app = rm1.submitApp(200);
    MockAM am1 = MockRM.launchAndRegisterAM(app, rm1, nm1);
    final ContainerId runningContainer =
        ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
    am1.allocate(null, Arrays.asList(runningContainer));

    Map schedulerApplications = scheduler.getSchedulerApplications();
    SchedulerApplication schedulerApp =
        (SchedulerApplication) scheduler.getSchedulerApplications().get(
            app.getApplicationId());
    schedulerApplications.put(mockAPp.getApplicationId(), application);

    scheduler.clearPendingContainerCache();

    Assert.assertEquals("Pending containers are not released "
        + "when one of the application attempt is null !", schedulerApp
        .getCurrentAppAttempt().getPendingRelease().size(), 0);
  } finally {
    if (rm1 != null) {
      rm1.stop();
    }
  }
}