Java YarnConfiguration类代码示例

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入依赖的package包/类
private void initAndStartStore(final FileSystem fs) throws IOException,
    URISyntaxException {
  Configuration conf = new Configuration();
  fs.initialize(new URI("/"), conf);
  fsWorkingPath =
      new Path("target",
        TestFileSystemApplicationHistoryStore.class.getSimpleName());
  fs.delete(fsWorkingPath, true);
  conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI,
    fsWorkingPath.toString());
  store = new FileSystemApplicationHistoryStore() {
    @Override
    protected FileSystem getFileSystem(Path path, Configuration conf) {
      return fs;
    }
  };
  store.init(conf);
  store.start();
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入依赖的package包/类
@Test
public void testUnhealthyNodeStatus() throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, hostFile
      .getAbsolutePath());

  rm = new MockRM(conf);
  rm.start();

  MockNM nm1 = rm.registerNode("host1:1234", 5120);
  Assert.assertEquals(0, ClusterMetrics.getMetrics().getUnhealthyNMs());
  // node healthy
  nm1.nodeHeartbeat(true);

  // node unhealthy
  nm1.nodeHeartbeat(false);
  checkUnealthyNMCount(rm, nm1, true, 1);

  // node healthy again
  nm1.nodeHeartbeat(true);
  checkUnealthyNMCount(rm, nm1, false, 0);
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入依赖的package包/类
@Test
public void testEmptyYarnClasspath() throws Exception {
  assumeNonMaprProfile();
  DacDaemonYarnApplication.Environment myEnv = new DacDaemonYarnApplication.Environment() {
    public String getEnv(String name) {
      return tempDir.getRoot().toString();
    }
  };

  YarnConfiguration yarnConfiguration = createYarnConfig("resource-manager", "hdfs://name-node:8020");
  yarnConfiguration.set(YarnDefaultsConfigurator.APP_CLASSPATH_JARS, YarnDefaultsConfigurator.MapRYarnDefaults.getAppClassPath());
  DacDaemonYarnApplication dacDaemonApp = new DacDaemonYarnApplication(DremioConfig.create(),
    yarnConfiguration, myEnv);

  List<String> names = dacDaemonApp.getJarNames();
  assertFalse(names.isEmpty());
  assertEquals(2, names.size());

  String flatNames = names.toString();
  assertTrue(flatNames.contains(SHIM_LOADER_NAME));
  assertTrue(flatNames.contains("maprfs-5.1.0-mapr.jar"));
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入依赖的package包/类
@Test
public void testContainerLaunch() throws Exception {
  Assume.assumeTrue(shouldRun());
  String expectedRunAsUser =
      conf.get(YarnConfiguration.NM_NONSECURE_MODE_LOCAL_USER_KEY,
        YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER);

  File touchFile = new File(workSpace, "touch-file");
  int ret = runAndBlock("touch", touchFile.getAbsolutePath());

  assertEquals(0, ret);
  FileStatus fileStatus =
      FileContext.getLocalFSFileContext().getFileStatus(
        new Path(touchFile.getAbsolutePath()));
  assertEquals(expectedRunAsUser, fileStatus.getOwner());
  cleanupAppFiles(expectedRunAsUser);

}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入依赖的package包/类
private YarnConfiguration createNMConfig() {
  YarnConfiguration conf = new YarnConfiguration();
  String localhostAddress = null;
  try {
    localhostAddress = InetAddress.getByName("localhost").getCanonicalHostName();
  } catch (UnknownHostException e) {
    Assert.fail("Unable to get localhost address: " + e.getMessage());
  }
  conf.setInt(YarnConfiguration.NM_PMEM_MB, 5 * 1024); // 5GB
  conf.set(YarnConfiguration.NM_ADDRESS, localhostAddress + ":12345");
  conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, localhostAddress + ":12346");
  conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
    remoteLogsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath());
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  return conf;
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入依赖的package包/类
static JobHistoryServer launchJobHistoryServer(String[] args) {
  Thread.
      setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
  StringUtils.startupShutdownMessage(JobHistoryServer.class, args, LOG);
  JobHistoryServer jobHistoryServer = null;
  try {
    jobHistoryServer = new JobHistoryServer();
    ShutdownHookManager.get().addShutdownHook(
        new CompositeServiceShutdownHook(jobHistoryServer),
        SHUTDOWN_HOOK_PRIORITY);
    YarnConfiguration conf = new YarnConfiguration(new JobConf());
    new GenericOptionsParser(conf, args);
    jobHistoryServer.init(conf);
    jobHistoryServer.start();
  } catch (Throwable t) {
    LOG.fatal("Error starting JobHistoryServer", t);
    ExitUtil.terminate(-1, "Error starting JobHistoryServer");
  }
  return jobHistoryServer;
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入依赖的package包/类
@Test
public void testAuxUnexpectedStop() {
  Configuration conf = new Configuration();
  conf.setStrings(YarnConfiguration.NM_AUX_SERVICES, new String[] { "Asrv", "Bsrv" });
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
      ServiceA.class, Service.class);
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"),
      ServiceB.class, Service.class);
  final AuxServices aux = new AuxServices();
  aux.init(conf);
  aux.start();

  Service s = aux.getServices().iterator().next();
  s.stop();
  assertEquals("Auxiliary service stopped, but AuxService unaffected.",
      STOPPED, aux.getServiceState());
  assertTrue(aux.getServices().isEmpty());
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入依赖的package包/类
protected ApplicationHistoryStore createApplicationHistoryStore(
    Configuration conf) {
  return ReflectionUtils.newInstance(conf.getClass(
    YarnConfiguration.APPLICATION_HISTORY_STORE,
    FileSystemApplicationHistoryStore.class,
    ApplicationHistoryStore.class), conf);
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入依赖的package包/类
@Test
public void testLaunchContainer() throws IOException {
  if (!shouldRun()) {
    LOG.warn("Docker not installed, aborting test.");
    return;
  }

  Map<String, String> env = new HashMap<String, String>();
  env.put(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, testImage);
  String touchFileName = "touch-file-" + System.currentTimeMillis();
  File touchFile = new File(dirsHandler.getLocalDirs().get(0), touchFileName);
  ContainerId cId = getNextContainerId();
  int ret = runAndBlock(
      cId, env, "touch", touchFile.getAbsolutePath(), "&&", "cp", touchFile.getAbsolutePath(), "/");

  assertEquals(0, ret);
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入依赖的package包/类
@Test
public void testCorruptedOwnerInfoForDomain() throws Exception {
  Configuration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  conf.set(YarnConfiguration.YARN_ADMIN_ACL, "owner");
  TimelineACLsManager timelineACLsManager =
      new TimelineACLsManager(conf);
  TimelineDomain domain = new TimelineDomain();
  try {
    timelineACLsManager.checkAccess(
        UserGroupInformation.createRemoteUser("owner"), domain);
    Assert.fail("Exception is expected");
  } catch (YarnException e) {
    Assert.assertTrue("It's not the exact expected exception", e.getMessage()
        .contains("is corrupted."));
  }
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入依赖的package包/类
private PlanFollower createPlanFollower() {
  String planFollowerPolicyClassName =
      conf.get(YarnConfiguration.RM_RESERVATION_SYSTEM_PLAN_FOLLOWER,
          getDefaultPlanFollower());
  if (planFollowerPolicyClassName == null) {
    return null;
  }
  LOG.info("Using PlanFollowerPolicy: " + planFollowerPolicyClassName);
  try {
    Class<?> planFollowerPolicyClazz =
        conf.getClassByName(planFollowerPolicyClassName);
    if (PlanFollower.class.isAssignableFrom(planFollowerPolicyClazz)) {
      return (PlanFollower) ReflectionUtils.newInstance(
          planFollowerPolicyClazz, conf);
    } else {
      throw new YarnRuntimeException("Class: " + planFollowerPolicyClassName
          + " not instance of " + PlanFollower.class.getCanonicalName());
    }
  } catch (ClassNotFoundException e) {
    throw new YarnRuntimeException(
        "Could not instantiate PlanFollowerPolicy: "
            + planFollowerPolicyClassName, e);
  }
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入依赖的package包/类
/**
 * If resource is public, verifyAccess should succeed
 */
@Test
public void testVerifyAccessPublicResource() throws Exception {
  Configuration conf = new Configuration();
  conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true);
  LocalResource resource = mock(LocalResource.class);
  // give public visibility
  when(resource.getVisibility()).thenReturn(LocalResourceVisibility.PUBLIC);
  Path localPath = mock(Path.class);
  when(localPath.getName()).thenReturn("foo.jar");
  String user = "joe";
  SCMUploaderProtocol scmClient = mock(SCMUploaderProtocol.class);
  FileSystem fs = mock(FileSystem.class);
  FileSystem localFs = FileSystem.getLocal(conf);
  SharedCacheUploader spied =
      createSpiedUploader(resource, localPath, user, conf, scmClient, fs,
          localFs);

  assertTrue(spied.verifyAccess());
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入依赖的package包/类
@Test
public void testAuxServiceRecoverySetup() throws IOException {
  Configuration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
  conf.set(YarnConfiguration.NM_RECOVERY_DIR, TEST_DIR.toString());
  conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
      new String[] { "Asrv", "Bsrv" });
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Asrv"),
      RecoverableServiceA.class, Service.class);
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT, "Bsrv"),
      RecoverableServiceB.class, Service.class);
  try {
    final AuxServices aux = new AuxServices();
    aux.init(conf);
    Assert.assertEquals(2, aux.getServices().size());
    File auxStorageDir = new File(TEST_DIR,
        AuxServices.STATE_STORE_ROOT_NAME);
    Assert.assertEquals(2, auxStorageDir.listFiles().length);
    aux.close();
  } finally {
    FileUtil.fullyDelete(TEST_DIR);
  }
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入依赖的package包/类
private void checkTags(Set<String> tags) {
  if (tags.size() > YarnConfiguration.APPLICATION_MAX_TAGS) {
    throw new IllegalArgumentException("Too many applicationTags, a maximum of only "
        + YarnConfiguration.APPLICATION_MAX_TAGS + " are allowed!");
  }
  for (String tag : tags) {
    if (tag.length() > YarnConfiguration.APPLICATION_MAX_TAG_LENGTH) {
      throw new IllegalArgumentException("Tag " + tag + " is too long, " +
          "maximum allowed length of a tag is " +
          YarnConfiguration.APPLICATION_MAX_TAG_LENGTH);
    }
    if (!CharMatcher.ASCII.matchesAllOf(tag)) {
      throw new IllegalArgumentException("A tag can only have ASCII " +
          "characters! Invalid tag - " + tag);
    }
  }
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入依赖的package包/类
@Test(timeout = 60000)
public void testLaunchWithArguments() throws Exception {
  ExitUtil.disableSystemExit();
  ApplicationHistoryServer historyServer = null;
  try {
    // Not able to modify the config of this test case,
    // but others have been customized to avoid conflicts
    String[] args = new String[2];
    args[0]="-D" + YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS + "=4000";
    args[1]="-D" + YarnConfiguration.TIMELINE_SERVICE_TTL_MS + "=200";
    historyServer =
        ApplicationHistoryServer.launchAppHistoryServer(args);
    Configuration conf = historyServer.getConfig();
    assertEquals("4000", conf.get(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_TTL_INTERVAL_MS));
    assertEquals("200", conf.get(YarnConfiguration.TIMELINE_SERVICE_TTL_MS));
  } catch (ExitUtil.ExitException e) {
    assertEquals(0, e.status);
    ExitUtil.resetFirstExitException();
    fail();
  } finally {
    if (historyServer != null) {
      historyServer.stop();
    }
  }
}