Java YarnConfiguration.set方法代码示例

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@Unstable
public static Text getTokenService(Configuration conf, String address,
    String defaultAddr, int defaultPort) {
  if (HAUtil.isHAEnabled(conf)) {
    // Build a list of service addresses to form the service name
    ArrayList<String> services = new ArrayList<String>();
    YarnConfiguration yarnConf = new YarnConfiguration(conf);
    for (String rmId : HAUtil.getRMHAIds(conf)) {
      // Set RM_ID to get the corresponding RM_ADDRESS
      yarnConf.set(YarnConfiguration.RM_HA_ID, rmId);
      services.add(SecurityUtil.buildTokenService(
          yarnConf.getSocketAddr(address, defaultAddr, defaultPort))
          .toString());
    }
    return new Text(Joiner.on(',').join(services));
  }

  // Non-HA case - no need to set RM_ID
  return SecurityUtil.buildTokenService(conf.getSocketAddr(address,
    defaultAddr, defaultPort));
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@Before
public void setup() throws IOException {
  failoverThread = null;
  keepRunning = true;
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
  conf.setInt(YarnConfiguration.CLIENT_FAILOVER_MAX_ATTEMPTS, 5);
  conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
  setRpcAddressForRM(RM1_NODE_ID, RM1_PORT_BASE);
  setRpcAddressForRM(RM2_NODE_ID, RM2_PORT_BASE);

  conf.setLong(YarnConfiguration.CLIENT_FAILOVER_SLEEPTIME_BASE_MS, 100L);

  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
  conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true);
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@BeforeClass
public static void setupServer() throws Exception {
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
  conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
      MemoryTimelineStore.class, TimelineStore.class);
  conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, "HTTPS_ONLY");

  File base = new File(BASEDIR);
  FileUtil.fullyDelete(base);
  base.mkdirs();
  keystoresDir = new File(BASEDIR).getAbsolutePath();
  sslConfDir =
      KeyStoreTestUtil.getClasspathDir(TestTimelineWebServicesWithSSL.class);

  KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false);
  conf.addResource("ssl-server.xml");
  conf.addResource("ssl-client.xml");

  timelineServer = new ApplicationHistoryServer();
  timelineServer.init(conf);
  timelineServer.start();
  store = timelineServer.getTimelineStore();
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
private void uploadDefaultConfiguration() throws IOException {
  Configuration conf = new Configuration();
  uploadConfiguration(conf, "core-site.xml");

  YarnConfiguration yarnConf = new YarnConfiguration();
  yarnConf.set(YarnConfiguration.RM_CONFIGURATION_PROVIDER_CLASS,
      "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider");
  uploadConfiguration(yarnConf, "yarn-site.xml");

  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration();
  uploadConfiguration(csConf, "capacity-scheduler.xml");

  Configuration hadoopPolicyConf = new Configuration(false);
  hadoopPolicyConf
      .addResource(YarnConfiguration.HADOOP_POLICY_CONFIGURATION_FILE);
  uploadConfiguration(hadoopPolicyConf, "hadoop-policy.xml");
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@Test
public void testEmptyYarnClasspath() throws Exception {
  assumeNonMaprProfile();
  DacDaemonYarnApplication.Environment myEnv = new DacDaemonYarnApplication.Environment() {
    public String getEnv(String name) {
      return tempDir.getRoot().toString();
    }
  };

  YarnConfiguration yarnConfiguration = createYarnConfig("resource-manager", "hdfs://name-node:8020");
  yarnConfiguration.set(YarnDefaultsConfigurator.APP_CLASSPATH_JARS, YarnDefaultsConfigurator.MapRYarnDefaults.getAppClassPath());
  DacDaemonYarnApplication dacDaemonApp = new DacDaemonYarnApplication(DremioConfig.create(),
    yarnConfiguration, myEnv);

  List<String> names = dacDaemonApp.getJarNames();
  assertFalse(names.isEmpty());
  assertEquals(2, names.size());

  String flatNames = names.toString();
  assertTrue(flatNames.contains(SHIM_LOADER_NAME));
  assertTrue(flatNames.contains("maprfs-5.1.0-mapr.jar"));
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@Before
public void setup() {
  privilegedOperationExecutorMock = mock(PrivilegedOperationExecutor.class);
  cGroupsHandlerMock = mock(CGroupsHandler.class);
  trafficControllerMock = mock(TrafficController.class);
  conf = new YarnConfiguration();
  tmpPath = new StringBuffer(System.getProperty("test.build.data")).append
      ('/').append("hadoop.tmp.dir").toString();
  device = YarnConfiguration.DEFAULT_NM_NETWORK_RESOURCE_INTERFACE;
  containerIdMock = mock(ContainerId.class);
  containerMock = mock(Container.class);
  when(containerIdMock.toString()).thenReturn(TEST_CONTAINER_ID_STR);
  //mock returning a mock - an angel died somewhere.
  when(containerMock.getContainerId()).thenReturn(containerIdMock);

  conf.setInt(YarnConfiguration
      .NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_MBIT, ROOT_BANDWIDTH_MBIT);
  conf.setInt(YarnConfiguration
      .NM_NETWORK_RESOURCE_OUTBOUND_BANDWIDTH_YARN_MBIT, YARN_BANDWIDTH_MBIT);
  conf.set("hadoop.tmp.dir", tmpPath);
  //In these tests, we'll only use TrafficController with recovery disabled
  conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, false);
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@Test 
public void testGetMasterUser() {
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(MRConfig.MASTER_USER_NAME, "foo");
  conf.set(YarnConfiguration.RM_PRINCIPAL, "bar");

  // default is yarn framework  
  assertEquals(Master.getMasterUserName(conf), "bar");

  // set framework name to classic
  conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
  assertEquals(Master.getMasterUserName(conf), "foo");

  // change framework to yarn
  conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
  assertEquals(Master.getMasterUserName(conf), "bar");

}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
private String buildRedirectPath() {
  // make a copy of the original configuration so not to mutate it. Also use
  // an YarnConfiguration to force loading of yarn-site.xml.
  YarnConfiguration yarnConf = new YarnConfiguration(rm.getConfig());
  String activeRMHAId = RMHAUtils.findActiveRMHAId(yarnConf);
  String path = "";
  if (activeRMHAId != null) {
    yarnConf.set(YarnConfiguration.RM_HA_ID, activeRMHAId);

    InetSocketAddress sock = YarnConfiguration.useHttps(yarnConf)
        ? yarnConf.getSocketAddr(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT)
        : yarnConf.getSocketAddr(YarnConfiguration.RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
            YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);

    path = sock.getHostName() + ":" + Integer.toString(sock.getPort());
    path = YarnConfiguration.useHttps(yarnConf)
        ? "https://" + path
        : "http://" + path;
  }
  return path;
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@BeforeClass
public static void createZKServer() throws Exception {
  File zkDir = new File("target/zookeeper");
  FileUtils.deleteDirectory(zkDir);
  assertTrue(zkDir.mkdirs());
  zookeeper = new MicroZookeeperService("InMemoryZKService");
  YarnConfiguration conf = new YarnConfiguration();
  conf.set(MicroZookeeperServiceKeys.KEY_ZKSERVICE_DIR, zkDir.getAbsolutePath());
  zookeeper.init(conf);
  zookeeper.start();
  addToTeardown(zookeeper);
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
public RMStateStore getRMStateStore() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  workingZnode = "/jira/issue/3077/rmstore";
  conf.set(YarnConfiguration.RM_ZK_ADDRESS, hostPort);
  conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode);
  this.client = createClient();
  this.store = new TestZKRMStateStoreInternal(conf, workingZnode);
  return this.store;
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@Before
public void setup() {
  conf = new YarnConfiguration();
  conf.set(YarnConfiguration.RM_SCHEDULER,
      "org.apache.hadoop.yarn.sls.scheduler.ResourceSchedulerWrapper");
  conf.set(SLSConfiguration.RM_SCHEDULER,
      "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
  conf.setBoolean(SLSConfiguration.METRICS_SWITCH, false);
  rm = new ResourceManager();
  rm.init(conf);
  rm.start();
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
private YarnConfiguration createYarnConfig(String rmHost, String fsURL) {
  YarnConfiguration yarnConfiguration = new YarnConfiguration();
  yarnConfiguration.set(FS_DEFAULT_NAME_KEY, fsURL);
  yarnConfiguration.set(RM_HOSTNAME, rmHost);
  yarnConfiguration.set(DremioConfig.LOCAL_WRITE_PATH_STRING, "/data/mydata");
  yarnConfiguration.set(DremioConfig.DIST_WRITE_PATH_STRING, "pdfs:///data/mydata/pdfs");
  yarnConfiguration.set(DremioConfig.YARN_JVM_OPTIONS, " -Xmx4096");
  yarnConfiguration.set(DacDaemonYarnApplication.YARN_CONTAINER_COUNT, "3");
  yarnConfiguration.set(DacDaemonYarnApplication.YARN_MEMORY_ON_HEAP, "4096");
  yarnConfiguration.set(DacDaemonYarnApplication.YARN_MEMORY_OFF_HEAP, "5120");
  yarnConfiguration.set(DacDaemonYarnApplication.YARN_CPU, "2");

  return yarnConfiguration;
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@Before
public void setup() throws Exception {
  fsPath = new File("target", getClass().getSimpleName() +
      "-tmpDir").getAbsoluteFile();
  fsContext = FileContext.getLocalFSFileContext();
  fsContext.delete(new Path(fsPath.getAbsolutePath()), true);
  conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_RECOVERY_ENABLED, true);
  conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
      LeveldbTimelineStateStore.class,
      TimelineStateStore.class);
  conf.set(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_STATE_STORE_PATH,
      fsPath.getAbsolutePath());
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
@Test
public void testStateStoreRemovalOnDecommission() throws IOException {
  final File recoveryDir = new File(basedir, "nm-recovery");
  nm = new TestNodeManager();
  YarnConfiguration conf = createNMConfig();
  conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
  conf.set(YarnConfiguration.NM_RECOVERY_DIR, recoveryDir.getAbsolutePath());

  // verify state store is not removed on normal shutdown
  nm.init(conf);
  nm.start();
  Assert.assertTrue(recoveryDir.exists());
  Assert.assertTrue(recoveryDir.isDirectory());
  nm.stop();
  nm = null;
  Assert.assertTrue(recoveryDir.exists());
  Assert.assertTrue(recoveryDir.isDirectory());

  // verify state store is removed on decommissioned shutdown
  nm = new TestNodeManager();
  nm.init(conf);
  nm.start();
  Assert.assertTrue(recoveryDir.exists());
  Assert.assertTrue(recoveryDir.isDirectory());
  nm.getNMContext().setDecommissioned(true);
  nm.stop();
  nm = null;
  Assert.assertFalse(recoveryDir.exists());
}
 

import org.apache.hadoop.yarn.conf.YarnConfiguration; //导入方法依赖的package包/类
private YarnConfiguration createNMConfig() {
  YarnConfiguration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.NM_PMEM_MB, 5*1024); // 5GB
  conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:12345");
  conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:12346");
  conf.set(YarnConfiguration.NM_LOG_DIRS, logsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogsDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, nmLocalDir.getAbsolutePath());
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  return conf;
}