Java MockRM.start方法代码示例

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test
public void testAsyncScheduling() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();

  final int NODES = 100;
  
  // Register nodes
  for (int i=0; i < NODES; ++i) {
    String host = "192.168.1." + i;
    RMNode node =
        MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
    cs.handle(new NodeAddedSchedulerEvent(node));
  }
  
  // Now directly exercise the scheduling loop
  for (int i=0; i < NODES; ++i) {
    CapacityScheduler.schedule(cs);
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test
public void testUpdateDemand() {
  conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
  resourceManager = new MockRM(conf);
  resourceManager.start();
  scheduler = (FairScheduler) resourceManager.getResourceScheduler();
  scheduler.allocConf = mock(AllocationConfiguration.class);

  String queueName = "root.queue1";
  when(scheduler.allocConf.getMaxResources(queueName)).thenReturn(maxResource);
  when(scheduler.allocConf.getMinResources(queueName)).thenReturn(Resources.none());
  FSLeafQueue schedulable = new FSLeafQueue(queueName, scheduler, null);

  FSAppAttempt app = mock(FSAppAttempt.class);
  Mockito.when(app.getDemand()).thenReturn(maxResource);

  schedulable.addAppSchedulable(app);
  schedulable.addAppSchedulable(app);

  schedulable.updateDemand();

  assertTrue("Demand is greater than max allowed ",
      Resources.equals(schedulable.getDemand(), maxResource));
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
private void testMinimumAllocation(YarnConfiguration conf, int testAlloc)
    throws Exception {
  MockRM rm = new MockRM(conf);
  rm.start();

  // Register node1
  MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);

  // Submit an application
  RMApp app1 = rm.submitApp(testAlloc);

  // kick the scheduling
  nm1.nodeHeartbeat(true);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
  am1.registerAppAttempt();
  SchedulerNodeReport report_nm1 =
      rm.getResourceScheduler().getNodeReport(nm1.getNodeId());

  int checkAlloc =
      conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
  Assert.assertEquals(checkAlloc, report_nm1.getUsedResource().getMemory());

  rm.stop();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
private MockRM initRM(Configuration conf) {
  MockRM rm = new MockRM(conf) {
    @Override
    public RMNodeLabelsManager createNodeLabelManager() {
      return lmgr;
    }
  };
  rm.getRMContext().setNodeLabelManager(lmgr);
  rm.start();
  Assert.assertEquals(Service.STATE.STARTED, rm.getServiceState());
  return rm;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test(timeout = 30000)
public void testAMContainerAllocationWhenDNSUnavailable() throws Exception {
  MockRM rm1 = new MockRM(conf) {
    @Override
    protected RMSecretManagerService createRMSecretManagerService() {
      return new TestRMSecretManagerService(conf, rmContext);
    }
  };
  rm1.start();

  MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000);
  SecurityUtilTestHelper.setTokenServiceUseIp(true);
  RMApp app1 = rm1.submitApp(200);
  RMAppAttempt attempt = app1.getCurrentAppAttempt();
  nm1.nodeHeartbeat(true);

  // fetching am container will fail, keep retrying 5 times.
  while (numRetries <= 5) {
    nm1.nodeHeartbeat(true);
    Thread.sleep(1000);
    Assert.assertEquals(RMAppAttemptState.SCHEDULED,
      attempt.getAppAttemptState());
    System.out.println("Waiting for am container to be allocated.");
  }

  SecurityUtilTestHelper.setTokenServiceUseIp(false);
  rm1.waitForState(attempt.getAppAttemptId(), RMAppAttemptState.ALLOCATED);
  MockRM.launchAndRegisterAM(app1, rm1, nm1);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test
public void testLogAggregationContextPassedIntoContainerToken()
    throws Exception {
  MockRM rm1 = new MockRM(conf);
  rm1.start();
  MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000);
  MockNM nm2 = rm1.registerNode("127.0.0.1:2345", 8000);
  // LogAggregationContext is set as null
  Assert
    .assertNull(getLogAggregationContextFromContainerToken(rm1, nm1, null));

  // create a not-null LogAggregationContext
  LogAggregationContext logAggregationContext =
      LogAggregationContext.newInstance(
        "includePattern", "excludePattern",
        "rolledLogsIncludePattern",
        "rolledLogsExcludePattern");
  LogAggregationContext returned =
      getLogAggregationContextFromContainerToken(rm1, nm2,
        logAggregationContext);
  Assert.assertEquals("includePattern", returned.getIncludePattern());
  Assert.assertEquals("excludePattern", returned.getExcludePattern());
  Assert.assertEquals("rolledLogsIncludePattern",
    returned.getRolledLogsIncludePattern());
  Assert.assertEquals("rolledLogsExcludePattern",
    returned.getRolledLogsExcludePattern());
  rm1.stop();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
private static void setupAndStartRM() throws Exception {
  Configuration rmconf = new Configuration();
  rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
    YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  String httpPrefix = "hadoop.http.authentication.";
  rmconf.setStrings(httpPrefix + "type", "kerberos");
  rmconf.set(httpPrefix + KerberosAuthenticationHandler.PRINCIPAL,
    httpSpnegoPrincipal);
  rmconf.set(httpPrefix + KerberosAuthenticationHandler.KEYTAB,
    httpSpnegoKeytabFile.getAbsolutePath());
  // use any file for signature secret
  rmconf.set(httpPrefix + AuthenticationFilter.SIGNATURE_SECRET + ".file",
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    "kerberos");
  rmconf.setBoolean(YarnConfiguration.RM_WEBAPP_DELEGATION_TOKEN_AUTH_FILTER,
    true);
  rmconf.set("hadoop.http.filter.initializers",
    AuthenticationFilterInitializer.class.getName());
  rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY,
    httpSpnegoPrincipal);
  rmconf.set(YarnConfiguration.RM_KEYTAB,
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY,
    httpSpnegoPrincipal);
  rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
    httpSpnegoKeytabFile.getAbsolutePath());
  rmconf.setBoolean("mockrm.webapp.enabled", true);
  rmconf.set("yarn.resourcemanager.proxyuser.client.hosts", "*");
  rmconf.set("yarn.resourcemanager.proxyuser.client.groups", "*");
  UserGroupInformation.setConfiguration(rmconf);
  rm = new MockRM(rmconf);
  rm.start();

}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test
public void testBlackListNodes() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();

  String host = "127.0.0.1";
  RMNode node =
      MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
  fs.handle(new NodeAddedSchedulerEvent(node));

  ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  createMockRMApp(appAttemptId, rm.getRMContext());

  SchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appId, "default",
        "user");
  fs.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  fs.handle(attemptEvent);

  // Verify the blacklist can be updated independent of requesting containers
  fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(),
      Collections.singletonList(host), null);
  Assert.assertTrue(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(), null,
      Collections.singletonList(host));
  Assert.assertFalse(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  rm.stop();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Before
public void setup() {
  Configuration conf = createConfiguration();
  resourceManager = new MockRM(conf);
  resourceManager.start();
  scheduler = (FairScheduler) resourceManager.getResourceScheduler();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test
public void testRMNodeLabelsMappingProviderConfiguration() {
  conf.unset(YarnConfiguration.RM_NODE_LABELS_PROVIDER_CONFIG);
  try {
    MockRM rm = new MockRM(conf);
    rm.init(conf);
    rm.start();
    Assert.fail("Expected an exception");
  } catch (Exception e) {
    // expected an exception
    Assert.assertTrue(e.getMessage().contains(
        "RMNodeLabelsMappingProvider should be configured"));
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test
public void testWithNodeLabelUpdateEnabled() throws Exception {
  conf.setLong(YarnConfiguration.RM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS,
      1000);
  MockRM rm = new MockRM(conf);
  rm.init(conf);
  rm.getRMContext().getRMDelegatedNodeLabelsUpdater().nodeLabelsUpdateInterval
      = 3 * 1000;
  rm.start();

  RMNodeLabelsManager mgr = rm.getRMContext().getNodeLabelManager();
  mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y"));

  NodeId nodeId = toNodeId("h1:1234");
  assertEquals(0, mgr.getLabelsOnNode(nodeId).size());
  updateNodeLabels(nodeId, "x");
  registerNode(rm, nodeId);
  Thread.sleep(4000);
  assertCollectionEquals(ImmutableSet.of("x"), mgr.getLabelsOnNode(nodeId));

  // Ensure that node labels are updated if NodeLabelsProvider
  // gives different labels
  updateNodeLabels(nodeId, "y");
  Thread.sleep(4000);
  assertCollectionEquals(ImmutableSet.of("y"), mgr.getLabelsOnNode(nodeId));

  rm.stop();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test(timeout = 30000)
public void testApplicationTypeLimit() throws Exception {
  Logger rootLogger = LogManager.getRootLogger();
  rootLogger.setLevel(Level.DEBUG);
  MockRM rm = new MockRM();
  rm.start();
  RMApp app1 =
      rm.submitApp(200, "name", "user",
        new HashMap<ApplicationAccessType, String>(), false, "default", -1,
        null, "MAPREDUCE-LENGTH-IS-20");
  Assert.assertEquals("MAPREDUCE-LENGTH-IS-", app1.getApplicationType());
  rm.stop();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test
public void testExistenceOfResourceRequestInRMContainer() throws Exception {
  Configuration conf = new Configuration();
  MockRM rm1 = new MockRM(conf);
  rm1.start();
  MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000);
  RMApp app1 = rm1.submitApp(1024);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  ResourceScheduler scheduler = rm1.getResourceScheduler();

  // request a container.
  am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
  ContainerId containerId2 = ContainerId.newContainerId(
      am1.getApplicationAttemptId(), 2);
  rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);

  // Verify whether list of ResourceRequest is present in RMContainer
  // while moving to ALLOCATED state
  Assert.assertNotNull(scheduler.getRMContainer(containerId2)
      .getResourceRequests());

  // Allocate container
  am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>())
      .getAllocatedContainers();
  rm1.waitForState(nm1, containerId2, RMContainerState.ACQUIRED);

  // After RMContainer moving to ACQUIRED state, list of ResourceRequest will
  // be empty
  Assert.assertNull(scheduler.getRMContainer(containerId2)
      .getResourceRequests());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test
public void testAppSubmissionWithoutDelegationToken() throws Exception {
  conf.setBoolean(YarnConfiguration.RM_PROXY_USER_PRIVILEGES_ENABLED, true);
  // create token2
  Text userText2 = new Text("user2");
  DelegationTokenIdentifier dtId2 =
      new DelegationTokenIdentifier(new Text("user2"), new Text("renewer2"),
        userText2);
  final Token<DelegationTokenIdentifier> token2 =
      new Token<DelegationTokenIdentifier>(dtId2.getBytes(),
        "password2".getBytes(), dtId2.getKind(), new Text("service2"));
  final MockRM rm = new TestSecurityMockRM(conf, null) {
    @Override
    protected DelegationTokenRenewer createDelegationTokenRenewer() {
      return new DelegationTokenRenewer() {
        @Override
        protected Token<?>[] obtainSystemTokensForUser(String user,
            final Credentials credentials) throws IOException {
          credentials.addToken(token2.getService(), token2);
          return new Token<?>[] { token2 };
        }
      };
    }
  };
  rm.start();

  // submit an app without delegationToken
  RMApp app = rm.submitApp(200);

  // wait for the new retrieved hdfs token.
  GenericTestUtils.waitFor(new Supplier<Boolean>() {
    public Boolean get() {
      return rm.getRMContext().getDelegationTokenRenewer()
        .getDelegationTokens().contains(token2);
    }
  }, 1000, 20000);

  // check nm can retrieve the token
  final MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
  nm1.registerNode();
  NodeHeartbeatResponse response = nm1.nodeHeartbeat(true);
  ByteBuffer tokenBuffer =
      response.getSystemCredentialsForApps().get(app.getApplicationId());
  Assert.assertNotNull(tokenBuffer);
  Credentials appCredentials = new Credentials();
  DataInputByteBuffer buf = new DataInputByteBuffer();
  tokenBuffer.rewind();
  buf.reset(tokenBuffer);
  appCredentials.readTokenStorageStream(buf);
  Assert.assertTrue(appCredentials.getAllTokens().contains(token2));
}
 

import org.apache.hadoop.yarn.server.resourcemanager.MockRM; //导入方法依赖的package包/类
@Test
public void testConcurrentAccess() {
  conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
  resourceManager = new MockRM(conf);
  resourceManager.start();
  scheduler = (FairScheduler) resourceManager.getResourceScheduler();

  String queueName = "root.queue1";
  final FSLeafQueue schedulable = scheduler.getQueueManager().
    getLeafQueue(queueName, true);
  ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
  RMContext rmContext = resourceManager.getRMContext();
  final FSAppAttempt app =
      new FSAppAttempt(scheduler, applicationAttemptId, "user1",
          schedulable, null, rmContext);

  // this needs to be in sync with the number of runnables declared below
  int testThreads = 2;
  List<Runnable> runnables = new ArrayList<Runnable>();

  // add applications to modify the list
  runnables.add(new Runnable() {
    @Override
    public void run() {
      for (int i=0; i < 500; i++) {
        schedulable.addAppSchedulable(app);
      }
    }
  });

  // iterate over the list a couple of times in a different thread
  runnables.add(new Runnable() {
    @Override
    public void run() {
      for (int i=0; i < 500; i++) {
        schedulable.getResourceUsage();
      }
    }
  });

  final List<Throwable> exceptions = Collections.synchronizedList(
      new ArrayList<Throwable>());
  final ExecutorService threadPool = Executors.newFixedThreadPool(
      testThreads);

  try {
    final CountDownLatch allExecutorThreadsReady =
        new CountDownLatch(testThreads);
    final CountDownLatch startBlocker = new CountDownLatch(1);
    final CountDownLatch allDone = new CountDownLatch(testThreads);
    for (final Runnable submittedTestRunnable : runnables) {
      threadPool.submit(new Runnable() {
        public void run() {
          allExecutorThreadsReady.countDown();
          try {
            startBlocker.await();
            submittedTestRunnable.run();
          } catch (final Throwable e) {
            exceptions.add(e);
          } finally {
            allDone.countDown();
          }
        }
      });
    }
    // wait until all threads are ready
    allExecutorThreadsReady.await();
    // start all test runners
    startBlocker.countDown();
    int testTimeout = 2;
    assertTrue("Timeout waiting for more than " + testTimeout + " seconds",
        allDone.await(testTimeout, TimeUnit.SECONDS));
  } catch (InterruptedException ie) {
    exceptions.add(ie);
  } finally {
    threadPool.shutdownNow();
  }
  assertTrue("Test failed with exception(s)" + exceptions,
      exceptions.isEmpty());
}