Java AppAddedSchedulerEvent类代码示例

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; //导入依赖的package包/类
public static SchedulerApplication<SchedulerApplicationAttempt>
    verifyAppAddedAndRemovedFromScheduler(
        Map<ApplicationId, SchedulerApplication<SchedulerApplicationAttempt>> applications,
        EventHandler<SchedulerEvent> handler, String queueName)
        throws Exception {
  ApplicationId appId =
      ApplicationId.newInstance(System.currentTimeMillis(), 1);
  AppAddedSchedulerEvent appAddedEvent =
      new AppAddedSchedulerEvent(appId, queueName, "user");
  handler.handle(appAddedEvent);
  SchedulerApplication<SchedulerApplicationAttempt> app =
      applications.get(appId);
  // verify application is added.
  Assert.assertNotNull(app);
  Assert.assertEquals("user", app.getUser());

  AppRemovedSchedulerEvent appRemoveEvent =
      new AppRemovedSchedulerEvent(appId, RMAppState.FINISHED);
  handler.handle(appRemoveEvent);
  Assert.assertNull(applications.get(appId));
  return app;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; //导入依赖的package包/类
@Test
public void testEmptyQueueName() throws Exception {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  // only default queue
  assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());

  // submit app with empty queue
  ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1);
  AppAddedSchedulerEvent appAddedEvent =
      new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "", "user1");
  scheduler.handle(appAddedEvent);

  // submission rejected
  assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
  assertNull(scheduler.getSchedulerApp(appAttemptId));
  assertEquals(0, resourceManager.getRMContext().getRMApps().size());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; //导入依赖的package包/类
/**
 * {@link RMAppAttemptState#SUBMITTED}
 */
private void testAppAttemptSubmittedState() {
  assertEquals(RMAppAttemptState.SUBMITTED, 
      applicationAttempt.getAppAttemptState());
  assertEquals(0, applicationAttempt.getDiagnostics().length());
  assertEquals(0,applicationAttempt.getJustFinishedContainers().size());
  assertNull(applicationAttempt.getMasterContainer());
  assertEquals(0.0, (double)applicationAttempt.getProgress(), 0.0001);
  assertEquals(0, applicationAttempt.getRanNodes().size());
  assertNull(applicationAttempt.getFinalApplicationStatus());
  
  // Check events
  verify(masterService).
      registerAppAttempt(applicationAttempt.getAppAttemptId());
  verify(scheduler).handle(any(AppAddedSchedulerEvent.class));
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; //导入依赖的package包/类
@Test
public void testUserAsDefaultQueue() throws Exception {
  Configuration conf = createConfiguration();
  conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
  scheduler.reinitialize(conf, resourceManager.getRMContext());
  AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(
      createAppAttemptId(1, 1), "default", "user1");
  scheduler.handle(appAddedEvent);
  assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1")
      .getAppSchedulables().size());
  assertEquals(0, scheduler.getQueueManager().getLeafQueue("default")
      .getAppSchedulables().size());

  conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false");
  scheduler.reinitialize(conf, resourceManager.getRMContext());
  AppAddedSchedulerEvent appAddedEvent2 = new AppAddedSchedulerEvent(
      createAppAttemptId(2, 1), "default", "user2");
  scheduler.handle(appAddedEvent2);
  assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1")
      .getAppSchedulables().size());
  assertEquals(1, scheduler.getQueueManager().getLeafQueue("default")
      .getAppSchedulables().size());
  assertEquals(0, scheduler.getQueueManager().getLeafQueue("user2")
      .getAppSchedulables().size());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; //导入依赖的package包/类
@Test
public void testAppAdditionAndRemoval() throws Exception {
  AppAddedSchedulerEvent appAddedEvent1 = new AppAddedSchedulerEvent(
      createAppAttemptId(1, 1), "default", "user1");
  scheduler.handle(appAddedEvent1);

  // Scheduler should have two queues (the default and the one created for user1)
  assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());

  // That queue should have one app
  assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1")
      .getAppSchedulables().size());

  AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent(
      createAppAttemptId(1, 1), RMAppAttemptState.FINISHED);

  // Now remove app
  scheduler.handle(appRemovedEvent1);

  // Queue should have no apps
  assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1")
      .getAppSchedulables().size());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; //导入依赖的package包/类
/**
 * {@link RMAppAttemptState#SUBMITTED}
 */
private void testAppAttemptSubmittedState() {
  assertEquals(RMAppAttemptState.SUBMITTED, 
      applicationAttempt.getAppAttemptState());
  assertEquals(0, applicationAttempt.getDiagnostics().length());
  assertEquals(0,applicationAttempt.getJustFinishedContainers().size());
  assertNull(applicationAttempt.getMasterContainer());
  assertEquals(0.0, (double)applicationAttempt.getProgress(), 0.0001);
  assertEquals(0, applicationAttempt.getRanNodes().size());
  assertNull(applicationAttempt.getFinalApplicationStatus());
  if (UserGroupInformation.isSecurityEnabled()) {
    verify(clientToAMTokenManager).createMasterKey(
        applicationAttempt.getAppAttemptId());
    // can't create ClientToken as at this time ClientTokenMasterKey has
    // not been registered in the SecretManager
    assertNull(applicationAttempt.createClientToken("some client"));
  }
  assertNull(applicationAttempt.createClientToken(null));
  assertNotNull(applicationAttempt.getAMRMToken());
  // Check events
  verify(masterService).
      registerAppAttempt(applicationAttempt.getAppAttemptId());
  verify(scheduler).handle(any(AppAddedSchedulerEvent.class));
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; //导入依赖的package包/类
@Test
public void testAppAdditionAndRemoval() throws Exception {
  AppAddedSchedulerEvent appAddedEvent1 = new AppAddedSchedulerEvent(
      createAppAttemptId(1, 1), "default", "user1");
  scheduler.handle(appAddedEvent1);

  // Scheduler should have two queues (the default and the one created for user1)
  assertEquals(2, scheduler.getQueueManager().getLeafQueues().size());

  // That queue should have one app
  assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
      .getAppSchedulables().size());

  AppRemovedSchedulerEvent appRemovedEvent1 = new AppRemovedSchedulerEvent(
      createAppAttemptId(1, 1), RMAppAttemptState.FINISHED);

  // Now remove app
  scheduler.handle(appRemovedEvent1);

  // Queue should have no apps
  assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true)
      .getAppSchedulables().size());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; //导入依赖的package包/类
@Override
public void transition(RMAppImpl app, RMAppEvent event) {
  if (event instanceof RMAppNewSavedEvent) {
    RMAppNewSavedEvent storeEvent = (RMAppNewSavedEvent) event;
    // For HA this exception needs to be handled by giving up
    // master status if we got fenced
    if (((RMAppNewSavedEvent) event).getStoredException() != null) {
      LOG.error(
        "Failed to store application: " + storeEvent.getApplicationId(),
        storeEvent.getStoredException());
      ExitUtil.terminate(1, storeEvent.getStoredException());
    }
  }
  app.handler.handle(new AppAddedSchedulerEvent(app.applicationId,
    app.submissionContext.getQueue(), app.user));
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; //导入依赖的package包/类
public static <T> SchedulerApplication verifyAppAddedAndRemovedFromScheduler(
    final Map<ApplicationId, SchedulerApplication> applications,
    EventHandler<SchedulerEvent> handler, String queueName) throws Exception {
  ApplicationId appId =
      ApplicationId.newInstance(System.currentTimeMillis(), 1);
  AppAddedSchedulerEvent appAddedEvent =
      new AppAddedSchedulerEvent(appId, queueName, "user");
  handler.handle(appAddedEvent);
  SchedulerApplication app = applications.get(appId);
  // verify application is added.
  Assert.assertNotNull(app);
  Assert.assertEquals("user", app.getUser());

  AppRemovedSchedulerEvent appRemoveEvent =
      new AppRemovedSchedulerEvent(appId, RMAppState.FINISHED);
  handler.handle(appRemoveEvent);
  Assert.assertNull(applications.get(appId));
  return app;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; //导入依赖的package包/类
@Test
public void testUserAsDefaultQueue() throws Exception {
  conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
  scheduler.reinitialize(conf, resourceManager.getRMContext());
  RMContext rmContext = resourceManager.getRMContext();
  Map<ApplicationId, RMApp> appsMap = rmContext.getRMApps();
  ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1);
  RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf,
      null, null, null, ApplicationSubmissionContext.newInstance(null, null,
      null, null, null, false, false, 0, null, null), null, null, 0, null, null);
  appsMap.put(appAttemptId.getApplicationId(), rmApp);
  
  AppAddedSchedulerEvent appAddedEvent =
      new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "default",
        "user1");
  scheduler.handle(appAddedEvent);
  AppAttemptAddedSchedulerEvent attempAddedEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attempAddedEvent);
  assertEquals(1, scheduler.getQueueManager().getLeafQueue("user1", true)
      .getRunnableAppSchedulables().size());
  assertEquals(0, scheduler.getQueueManager().getLeafQueue("default", true)
      .getRunnableAppSchedulables().size());
  assertEquals("root.user1", rmApp.getQueue());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; //导入依赖的package包/类
@Test
public void testNotUserAsDefaultQueue() throws Exception {
  conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "false");
  scheduler.reinitialize(conf, resourceManager.getRMContext());
  RMContext rmContext = resourceManager.getRMContext();
  Map<ApplicationId, RMApp> appsMap = rmContext.getRMApps();
  ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1);
  RMApp rmApp = new RMAppImpl(appAttemptId.getApplicationId(), rmContext, conf,
      null, null, null, ApplicationSubmissionContext.newInstance(null, null,
      null, null, null, false, false, 0, null, null), null, null, 0, null, null);
  appsMap.put(appAttemptId.getApplicationId(), rmApp);

  AppAddedSchedulerEvent appAddedEvent =
      new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "default",
        "user2");
  scheduler.handle(appAddedEvent);
  AppAttemptAddedSchedulerEvent attempAddedEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attempAddedEvent);
  assertEquals(0, scheduler.getQueueManager().getLeafQueue("user1", true)
      .getRunnableAppSchedulables().size());
  assertEquals(1, scheduler.getQueueManager().getLeafQueue("default", true)
      .getRunnableAppSchedulables().size());
  assertEquals(0, scheduler.getQueueManager().getLeafQueue("user2", true)
      .getRunnableAppSchedulables().size());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; //导入依赖的package包/类
@Test
public void testEmptyQueueName() throws Exception {
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  // only default queue
  assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());

  // submit app with empty queue
  ApplicationAttemptId appAttemptId = createAppAttemptId(1, 1);
  AppAddedSchedulerEvent appAddedEvent =
      new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "", "user1");
  scheduler.handle(appAddedEvent);

  // submission rejected
  assertEquals(1, scheduler.getQueueManager().getLeafQueues().size());
  assertNull(scheduler.getSchedulerApp(appAttemptId));
  assertEquals(0, resourceManager.getRMContext().getRMApps().size());
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; //导入依赖的package包/类
@SuppressWarnings("deprecation")
public synchronized void submit() throws IOException, YarnException {
  ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  context.setApplicationId(this.applicationId);
  context.setQueue(this.queue);
  
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
  context.setAMContainerSpec(amContainer);
  context.setResource(Resources.createResource(
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
  
  SubmitApplicationRequest request = recordFactory
      .newRecordInstance(SubmitApplicationRequest.class);
  request.setApplicationSubmissionContext(context);
  final ResourceScheduler scheduler = resourceManager.getResourceScheduler();
  
  resourceManager.getClientRMService().submitApplication(request);

  // Notify scheduler
  AppAddedSchedulerEvent addAppEvent =
      new AppAddedSchedulerEvent(this.applicationId, this.queue, "user");
  scheduler.handle(addAppEvent);
  AppAttemptAddedSchedulerEvent addAttemptEvent =
      new AppAttemptAddedSchedulerEvent(this.applicationAttemptId, false);
  scheduler.handle(addAttemptEvent);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; //导入依赖的package包/类
@Test(timeout=5000)
public void testAppAttemptMetrics() throws Exception {
  AsyncDispatcher dispatcher = new InlineDispatcher();
  
  FifoScheduler scheduler = new FifoScheduler();
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  RMContext rmContext = new RMContextImpl(dispatcher, null,
      null, null, null, null, null, null, null, writer, scheduler);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(
      mock(SystemMetricsPublisher.class));

  Configuration conf = new Configuration();
  scheduler.setRMContext(rmContext);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rmContext);
  QueueMetrics metrics = scheduler.getRootQueueMetrics();
  int beforeAppsSubmitted = metrics.getAppsSubmitted();

  ApplicationId appId = BuilderUtils.newApplicationId(200, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");
  scheduler.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent);

  appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);
  SchedulerEvent attemptEvent2 =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent2);

  int afterAppsSubmitted = metrics.getAppsSubmitted();
  Assert.assertEquals(1, afterAppsSubmitted - beforeAppsSubmitted);
  scheduler.stop();
}
 

import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent; //导入依赖的package包/类
@Test
public void testBlackListNodes() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();

  String host = "127.0.0.1";
  RMNode node =
      MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
  fs.handle(new NodeAddedSchedulerEvent(node));

  ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  createMockRMApp(appAttemptId, rm.getRMContext());

  SchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appId, "default",
        "user");
  fs.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  fs.handle(attemptEvent);

  // Verify the blacklist can be updated independent of requesting containers
  fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(),
      Collections.singletonList(host), null);
  Assert.assertTrue(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(), null,
      Collections.singletonList(host));
  Assert.assertFalse(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  rm.stop();
}