Java ApplicationConstants类代码示例

import org.apache.hadoop.yarn.api.ApplicationConstants; //导入依赖的package包/类
private String makeContainerCommand(long containerMemory, String clusterSpec,
    String jobName, int taskIndex) {
  String[] commands = new String[]{
      ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java",
      "-Xmx" + containerMemory + "m",
      TFServerRunner.class.getName() + " ",
      Utils.mkOption(Constants.OPT_CLUSTER_SPEC, clusterSpec),
      Utils.mkOption(Constants.OPT_JOB_NAME, jobName),
      Utils.mkOption(Constants.OPT_TASK_INDEX, taskIndex),
      "1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/TFServerRunner." +
          ApplicationConstants.STDOUT,
      "2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/TFServerRunner." +
          ApplicationConstants.STDERR
  };

  return Utils.mkString(commands, " ");
}
 

import org.apache.hadoop.yarn.api.ApplicationConstants; //导入依赖的package包/类
private String makeAppMasterCommand(String tfLib, String tfJar) {
  String[] commands = new String[]{
      ApplicationConstants.Environment.JAVA_HOME.$$() + "/bin/java",
      // Set Xmx based on am memory size
      "-Xmx" + amMemory + "m",
      // Set class name
      ApplicationMaster.class.getName(),
      Utils.mkOption(Constants.OPT_TF_CONTAINER_MEMORY, containerMemory),
      Utils.mkOption(Constants.OPT_TF_CONTAINER_VCORES, containerVCores),
      Utils.mkOption(Constants.OPT_TF_WORKER_NUM, workerNum),
      Utils.mkOption(Constants.OPT_TF_PS_NUM, psNum),
      Utils.mkOption(Constants.OPT_TF_LIB, tfLib),
      Utils.mkOption(Constants.OPT_TF_JAR, tfJar),
      "1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout",
      "2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"
  };
  return Utils.mkString(commands, " ");
}
 

import org.apache.hadoop.yarn.api.ApplicationConstants; //导入依赖的package包/类
public static Map<String, String> setJavaEnv(Configuration conf) {
  Map<String, String> env = new HashMap<>();

  StringBuilder classPathEnv = new StringBuilder(ApplicationConstants.Environment.CLASSPATH.$$())
      .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*");

  for (String c : conf.getStrings(
      YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) {
    classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR);
    classPathEnv.append(c.trim());
  }

  if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
    classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR);
    classPathEnv.append(System.getProperty("java.class.path"));
  }

  env.put("CLASSPATH", classPathEnv.toString());
  return env;
}
 

import org.apache.hadoop.yarn.api.ApplicationConstants; //导入依赖的package包/类
@VisibleForTesting
public static String expandEnvironment(String var,
    Path containerLogDir) {
  var = var.replace(ApplicationConstants.LOG_DIR_EXPANSION_VAR,
    containerLogDir.toString());
  var =  var.replace(ApplicationConstants.CLASS_PATH_SEPARATOR,
    File.pathSeparator);

  // replace parameter expansion marker. e.g. {{VAR}} on Windows is replaced
  // as %VAR% and on Linux replaced as "$VAR"
  if (Shell.WINDOWS) {
    var = var.replaceAll("(\\{\\{)|(\\}\\})", "%");
  } else {
    var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_LEFT, "$");
    var = var.replace(ApplicationConstants.PARAMETER_EXPANSION_RIGHT, "");
  }
  return var;
}
 

import org.apache.hadoop.yarn.api.ApplicationConstants; //导入依赖的package包/类
@Test(timeout = 10000)
public void testEnvExpansion() throws IOException {
  Path logPath = new Path("/nm/container/logs");
  String input =
      Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/lib/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_LOG_HOME")
          + ApplicationConstants.LOG_DIR_EXPANSION_VAR;

  String res = ContainerLaunch.expandEnvironment(input, logPath);

  if (Shell.WINDOWS) {
    Assert.assertEquals("%HADOOP_HOME%/share/hadoop/common/*;"
        + "%HADOOP_HOME%/share/hadoop/common/lib/*;"
        + "%HADOOP_LOG_HOME%/nm/container/logs", res);
  } else {
    Assert.assertEquals("$HADOOP_HOME/share/hadoop/common/*:"
        + "$HADOOP_HOME/share/hadoop/common/lib/*:"
        + "$HADOOP_LOG_HOME/nm/container/logs", res);
  }
  System.out.println(res);
}
 

import org.apache.hadoop.yarn.api.ApplicationConstants; //导入依赖的package包/类
@Test (timeout = 120000)
public void testSetClasspathWithUserPrecendence() {
   Configuration conf = new Configuration();
   conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
   conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
   Map<String, String> env = new HashMap<String, String>();
   try {
     MRApps.setClasspath(env, conf);
   } catch (Exception e) {
     fail("Got exception while setting classpath");
   }
   String env_str = env.get("CLASSPATH");
   String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
     Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar",
       "job.jar/classes/", "job.jar/lib/*",
       ApplicationConstants.Environment.PWD.$$() + "/*"));
   assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST set, but not taking effect!",
     env_str.startsWith(expectedClasspath));
 }
 

import org.apache.hadoop.yarn.api.ApplicationConstants; //导入依赖的package包/类
@Test (timeout = 120000)
public void testSetClasspathWithNoUserPrecendence() {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, false);
  Map<String, String> env = new HashMap<String, String>();
  try {
    MRApps.setClasspath(env, conf);
  } catch (Exception e) {
    fail("Got exception while setting classpath");
  }
  String env_str = env.get("CLASSPATH");
  String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not in"
    + " the classpath!", env_str.contains(expectedClasspath));
  assertFalse("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!",
    env_str.startsWith(expectedClasspath));
}
 

import org.apache.hadoop.yarn.api.ApplicationConstants; //导入依赖的package包/类
@Test (timeout = 120000)
public void testSetClasspathWithJobClassloader() throws IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM, true);
  conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true);
  Map<String, String> env = new HashMap<String, String>();
  MRApps.setClasspath(env, conf);
  String cp = env.get("CLASSPATH");
  String appCp = env.get("APP_CLASSPATH");
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the"
    + " classpath!", cp.contains("jar" + ApplicationConstants.CLASS_PATH_SEPARATOR + "job"));
  assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!",
    cp.contains("PWD"));
  String expectedAppClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
    Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar",
      "job.jar/classes/", "job.jar/lib/*",
      ApplicationConstants.Environment.PWD.$$() + "/*"));
  assertEquals("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app"
    + " classpath!", expectedAppClasspath, appCp);
}
 

import org.apache.hadoop.yarn.api.ApplicationConstants; //导入依赖的package包/类
private synchronized void launchDummyTask(Container container){
    ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
    String new_command = "./launcher.py";
    String cmd = new_command + " 1>"
        + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"
        + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR
        + "/stderr";
    ctx.setCommands(Collections.singletonList(cmd));
    ctx.setTokens(setupTokens());
    ctx.setLocalResources(this.workerResources);
    synchronized (this){
        this.nmClient.startContainerAsync(container, ctx);
    }
}
 

import org.apache.hadoop.yarn.api.ApplicationConstants; //导入依赖的package包/类
public ContainerLaunchContext buildContainerContext(Map<String, LocalResource> localResources, YacopConfig yacopConfig) {
ContainerLaunchContext ctx = null;
  try {
    List<String> commands = new ArrayList<>();
    //cmd
    Vector<CharSequence> vargs = new Vector<>(5);
    vargs.add("(" + yacopConfig.getCmd() + ")");
    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout");
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr");
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
      command.append(str).append(" ");
    }
    commands.add(command.toString());
    //tokens
    Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials();
    DataOutputBuffer dob = new DataOutputBuffer();
    credentials.writeTokenStorageToStream(dob);
    ByteBuffer allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
    //ctx
    ctx = ContainerLaunchContext.newInstance(localResources, null, commands, null, allTokens.duplicate(), null);
    } catch (IOException e) {
      e.printStackTrace();
    }
    return ctx;
}
 

import org.apache.hadoop.yarn.api.ApplicationConstants; //导入依赖的package包/类
public List<String> prepareCommands() {
    Vector<CharSequence> vargs = new Vector<CharSequence>(30);
    vargs.add(Environment.JAVA_HOME.$$() + "/bin/java");
    vargs.add("-Xmx128m");
    vargs.add(appMasterMainClass);
    vargs.add(">" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout");
//    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr");
    vargs.add("2>&1");
    StringBuilder command = new StringBuilder();
    for (CharSequence str : vargs) {
      command.append(str).append(" ");
    }

    List<String> commands = new ArrayList<String>();
    commands.add(command.toString());

    return commands;
  }
 

import org.apache.hadoop.yarn.api.ApplicationConstants; //导入依赖的package包/类
@GET
@Path(PATH_PHYSICAL_PLAN_CONTAINERS + "/{containerId}")
@Produces(MediaType.APPLICATION_JSON)
public JSONObject getContainer(@PathParam("containerId") String containerId) throws Exception
{
  init();
  ContainerInfo ci = null;
  if (containerId.equals(System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString()))) {
    ci = dagManager.getAppMasterContainerInfo();
  } else {
    for (ContainerInfo containerInfo : dagManager.getCompletedContainerInfo()) {
      if (containerInfo.id.equals(containerId)) {
        ci = containerInfo;
      }
    }
    if (ci == null) {
      StreamingContainerAgent sca = dagManager.getContainerAgent(containerId);
      if (sca == null) {
        throw new NotFoundException();
      }
      ci = sca.getContainerInfo();
    }
  }
  return new JSONObject(objectMapper.writeValueAsString(ci));
}
 

import org.apache.hadoop.yarn.api.ApplicationConstants; //导入依赖的package包/类
/**
 * Get environment variables in a {@link java.util.Map} used when launching a Yarn container.
 *
 * @param yarnConfiguration a Hadoop {@link Configuration} object carrying Hadoop/Yarn configuration properties
 * @return a {@link java.util.Map} storing environment variables used when launching a Yarn container
 */
public static Map<String, String> getEnvironmentVariables(Configuration yarnConfiguration) {
  Map<String, String> environmentVariableMap = Maps.newHashMap();

  Apps.addToEnvironment(environmentVariableMap, ApplicationConstants.Environment.JAVA_HOME.key(),
      System.getenv(ApplicationConstants.Environment.JAVA_HOME.key()));

  // Add jars/files in the working directory of the ApplicationMaster to the CLASSPATH
  Apps.addToEnvironment(environmentVariableMap, ApplicationConstants.Environment.CLASSPATH.key(),
      ApplicationConstants.Environment.PWD.$());
  Apps.addToEnvironment(environmentVariableMap, ApplicationConstants.Environment.CLASSPATH.key(),
      ApplicationConstants.Environment.PWD.$() + File.separator + "*");

  String[] classpaths = yarnConfiguration.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH);
  if (classpaths != null) {
    for (String classpath : classpaths) {
      Apps.addToEnvironment(
          environmentVariableMap, ApplicationConstants.Environment.CLASSPATH.key(), classpath.trim());
    }
  }

  return environmentVariableMap;
}
 

import org.apache.hadoop.yarn.api.ApplicationConstants; //导入依赖的package包/类
@Override
protected void startUp() throws Exception {
  if (this.jobConfPackagePath.isPresent()) {
    File path = new File(this.jobConfPackagePath.get());
    File jobConfigDir = new File(System.getenv().get(ApplicationConstants.Environment.PWD.key()), path.getName());
    if (jobConfigDir.exists()) {
      LOGGER.info("Loading job configurations from " + jobConfigDir);
      Properties properties = new Properties();
      properties.setProperty(ConfigurationKeys.JOB_CONFIG_FILE_DIR_KEY, jobConfigDir.getAbsolutePath());
      List<Properties> jobConfigs = SchedulerUtils.loadJobConfigs(properties);
      LOGGER.info("Loaded " + jobConfigs.size() + " job configuration(s)");
      for (Properties config : jobConfigs) {
        postNewJobConfigArrival(config.getProperty(ConfigurationKeys.JOB_NAME_KEY), config);
      }
    } else {
      LOGGER.warn("Job configuration directory " + jobConfigDir + " not found");
    }
  }
}
 

import org.apache.hadoop.yarn.api.ApplicationConstants; //导入依赖的package包/类
@Test (timeout = 120000)
public void testSetClasspath() throws IOException {
  Job job = Job.getInstance();
  Map<String, String> environment = new HashMap<String, String>();
  MRApps.setClasspath(environment, job.getConfiguration());
  assertTrue(environment.get("CLASSPATH").startsWith(
    ApplicationConstants.Environment.PWD.$() + File.pathSeparator));
  String yarnAppClasspath = 
      job.getConfiguration().get(
          YarnConfiguration.YARN_APPLICATION_CLASSPATH);
  if (yarnAppClasspath != null) {
    yarnAppClasspath = yarnAppClasspath.replaceAll(",\\s*", File.pathSeparator)
      .trim();
  }
  assertTrue(environment.get("CLASSPATH").contains(yarnAppClasspath));
  String mrAppClasspath = 
      job.getConfiguration().get(MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH);
  if (mrAppClasspath != null) {
    mrAppClasspath = mrAppClasspath.replaceAll(",\\s*", File.pathSeparator)
      .trim();
  }
  assertTrue(environment.get("CLASSPATH").contains(mrAppClasspath));
}