Java YarnConfiguration.setClass方法代码示例

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@Test
public void testContainerExecutorInitCall() {
  NodeManager nm = new NodeManager();
  YarnConfiguration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR,
      InvalidContainerExecutor.class,
      ContainerExecutor.class);
  try {
    nm.init(conf);
    fail("Init should fail");
  } catch (YarnRuntimeException e) {
    //PASS
    assert(e.getCause().getMessage().contains("dummy executor init called"));
  } finally {
    nm.stop();
  }
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@BeforeClass
public static void setup() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
  conf.setBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED, true);
  conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
      MemoryTimelineStore.class, TimelineStore.class);
  conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
      MemoryTimelineStateStore.class, TimelineStateStore.class);
  conf.setInt(
      YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE,
      2);

  timelineServer = new ApplicationHistoryServer();
  timelineServer.init(conf);
  timelineServer.start();
  store = timelineServer.getTimelineStore();

  metricsPublisher = new SystemMetricsPublisher();
  metricsPublisher.init(conf);
  metricsPublisher.start();
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager() {
    @Override
    protected RMNodeLabelsManager createNodeLabelManager() {
      RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
      mgr.init(getConfig());
      return mgr;
    }
  };
  CapacitySchedulerConfiguration csConf 
     = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      CapacityScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@BeforeClass
public static void setupServer() throws Exception {
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
  conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
      MemoryTimelineStore.class, TimelineStore.class);
  conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, "HTTPS_ONLY");

  File base = new File(BASEDIR);
  FileUtil.fullyDelete(base);
  base.mkdirs();
  keystoresDir = new File(BASEDIR).getAbsolutePath();
  sslConfDir =
      KeyStoreTestUtil.getClasspathDir(TestTimelineWebServicesWithSSL.class);

  KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false);
  conf.addResource("ssl-server.xml");
  conf.addResource("ssl-client.xml");

  timelineServer = new ApplicationHistoryServer();
  timelineServer.init(conf);
  timelineServer.start();
  store = timelineServer.getTimelineStore();
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
    ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@Override
protected void configureServlets() {
  bind(JAXBContextResolver.class);
  bind(RMWebServices.class);
  bind(GenericExceptionHandler.class);
  csConf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
  ResourceScheduler.class);
  rm = new MockRM(conf);
  bind(ResourceManager.class).toInstance(rm);
  serve("/*").with(GuiceContainer.class);
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@Test
public void testMetricsInitializedOnRMInit() {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER,
    FifoScheduler.class, ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
  checkApps(metrics, 0, 0, 0, 0, 0, 0, true);
  MetricsAsserts.assertGauge("ReservedContainers", 0, metrics);
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@Before
public void setUp() throws Exception {
  conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  mgr = new NullRMNodeLabelsManager();
  mgr.init(conf);
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@Before
public void setUp() throws Exception {
  conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoSchedulerWithMove.class,
      FifoSchedulerWithMove.class);
  conf.set(YarnConfiguration.YARN_ADMIN_ACL, " ");
  conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  resourceManager = new ResourceManager();
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  resourceManager.start();
  failMove = false;
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@Before
public void setup() throws Exception {
  fsPath = new File("target", getClass().getSimpleName() +
      "-tmpDir").getAbsoluteFile();
  fsContext = FileContext.getLocalFSFileContext();
  fsContext.delete(new Path(fsPath.getAbsolutePath()), true);
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_RECOVERY_ENABLED, true);
  conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
      LeveldbTimelineStateStore.class,
      TimelineStateStore.class);
  conf.set(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH,
      fsPath.getAbsolutePath());
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@BeforeClass
public static void setup() {
  conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      FifoScheduler.class, ResourceScheduler.class);
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@BeforeClass
public static void setup() {
  conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@Test(timeout = 3000000)
public void testResourceTypes() throws Exception {
  HashMap<YarnConfiguration, EnumSet<SchedulerResourceTypes>> driver =
      new HashMap<YarnConfiguration, EnumSet<SchedulerResourceTypes>>();

  CapacitySchedulerConfiguration csconf =
      new CapacitySchedulerConfiguration();
  csconf.setResourceComparator(DominantResourceCalculator.class);
  YarnConfiguration testCapacityDRConf = new YarnConfiguration(csconf);
  testCapacityDRConf.setClass(YarnConfiguration.RM_SCHEDULER,
    CapacityScheduler.class, ResourceScheduler.class);
  YarnConfiguration testCapacityDefConf = new YarnConfiguration();
  testCapacityDefConf.setClass(YarnConfiguration.RM_SCHEDULER,
    CapacityScheduler.class, ResourceScheduler.class);
  YarnConfiguration testFairDefConf = new YarnConfiguration();
  testFairDefConf.setClass(YarnConfiguration.RM_SCHEDULER,
    FairScheduler.class, ResourceScheduler.class);

  driver.put(conf, EnumSet.of(SchedulerResourceTypes.MEMORY));
  driver.put(testCapacityDRConf,
    EnumSet.of(SchedulerResourceTypes.CPU, SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.GPU));
  driver.put(testCapacityDefConf, EnumSet.of(SchedulerResourceTypes.MEMORY));
  driver.put(testFairDefConf,
    EnumSet.of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU, SchedulerResourceTypes.GPU));

  for (Map.Entry<YarnConfiguration, EnumSet<SchedulerResourceTypes>> entry : driver
    .entrySet()) {
    EnumSet<SchedulerResourceTypes> expectedValue = entry.getValue();
    MockRM rm = new MockRM(entry.getKey());
    rm.start();
    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
    RMApp app1 = rm.submitApp(2048);
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
    MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
    RegisterApplicationMasterResponse resp = am1.registerAppAttempt();
    EnumSet<SchedulerResourceTypes> types = resp.getSchedulerResourceTypes();
    LOG.info("types = " + types.toString());
    Assert.assertEquals(expectedValue, types);
    rm.stop();
  }
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@Test(timeout = 20000)
public void testPreemptedAMRestartOnRMRestart() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
  conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);

  conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
  // explicitly set max-am-retry count as 1.
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  MockRM rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1 = rm1.submitApp(200);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  CapacityScheduler scheduler =
      (CapacityScheduler) rm1.getResourceScheduler();
  ContainerId amContainer =
      ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);

  // Forcibly preempt the am container;
  scheduler.killContainer(scheduler.getRMContainer(amContainer));

  am1.waitForState(RMAppAttemptState.FAILED);
  Assert.assertTrue(! attempt1.shouldCountTowardsMaxAttemptRetry());
  rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);

  // state store has 1 attempt stored.
  ApplicationStateData appState =
      memStore.getState().getApplicationState().get(app1.getApplicationId());
  Assert.assertEquals(1, appState.getAttemptCount());
  // attempt stored has the preempted container exit status.
  Assert.assertEquals(ContainerExitStatus.PREEMPTED,
    appState.getAttempt(am1.getApplicationAttemptId())
      .getAMContainerExitStatus());
  // Restart rm.
  MockRM rm2 = new MockRM(conf, memStore);
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  nm1.registerNode();
  rm2.start();

  // Restarted RM should re-launch the am.
  MockAM am2 =
      rm2.waitForNewAMToLaunchAndRegister(app1.getApplicationId(), 2, nm1);
  MockRM.finishAMAndVerifyAppState(app1, rm2, nm1, am2);
  RMAppAttempt attempt2 =
      rm2.getRMContext().getRMApps().get(app1.getApplicationId())
        .getCurrentAppAttempt();
  Assert.assertTrue(attempt2.shouldCountTowardsMaxAttemptRetry());
  Assert.assertEquals(ContainerExitStatus.INVALID,
    appState.getAttempt(am2.getApplicationAttemptId())
      .getAMContainerExitStatus());
  rm1.stop();
  rm2.stop();
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@Test(timeout = 50000)
public void testRMRestartOrFailoverNotCountedForAMFailures()
    throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  conf.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
  conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);

  conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
  // explicitly set max-am-retry count as 1.
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  MockRM rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8000, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1 = rm1.submitApp(200);
  // AM should be restarted even though max-am-attempt is 1.
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
  Assert.assertTrue(((RMAppAttemptImpl) attempt1).mayBeLastAttempt());

  // Restart rm.
  MockRM rm2 = new MockRM(conf, memStore);
  rm2.start();
  ApplicationStateData appState =
      memStore.getState().getApplicationState().get(app1.getApplicationId());
  // re-register the NM
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  NMContainerStatus status = Records.newRecord(NMContainerStatus.class);
  status
    .setContainerExitStatus(ContainerExitStatus.KILLED_BY_RESOURCEMANAGER);
  status.setContainerId(attempt1.getMasterContainer().getId());
  status.setContainerState(ContainerState.COMPLETE);
  status.setDiagnostics("");
  nm1.registerNode(Collections.singletonList(status), null);

  rm2.waitForState(attempt1.getAppAttemptId(), RMAppAttemptState.FAILED);
  Assert.assertEquals(ContainerExitStatus.KILLED_BY_RESOURCEMANAGER,
    appState.getAttempt(am1.getApplicationAttemptId())
      .getAMContainerExitStatus());
  // Will automatically start a new AppAttempt in rm2
  rm2.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
  MockAM am2 =
      rm2.waitForNewAMToLaunchAndRegister(app1.getApplicationId(), 2, nm1);
  MockRM.finishAMAndVerifyAppState(app1, rm2, nm1, am2);
  RMAppAttempt attempt3 =
      rm2.getRMContext().getRMApps().get(app1.getApplicationId())
        .getCurrentAppAttempt();
  Assert.assertTrue(attempt3.shouldCountTowardsMaxAttemptRetry());
  Assert.assertEquals(ContainerExitStatus.INVALID,
    appState.getAttempt(am2.getApplicationAttemptId())
      .getAMContainerExitStatus());

  rm1.stop();
  rm2.stop();
}