Java Apps类代码示例

import org.apache.hadoop.yarn.util.Apps; //导入依赖的package包/类
private void setupAppMasterEnv(Map<String, String> appMasterEnv, String exclasspath) {
	for (String c: conf.getStrings(
			YarnConfiguration.YARN_APPLICATION_CLASSPATH, 
			YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
		Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), c.trim());
	}
	Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(),
			Environment.PWD.$() + File.separator +"*");
	
	System.out.println("##env pwd:"+Environment.PWD.$());
	/*
	String[] excparray = exclasspath.split(",");
	for (String cp: excparray) {
		Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), cp);
	}*/
	
}
 

import org.apache.hadoop.yarn.util.Apps; //导入依赖的package包/类
@Test(timeout = 10000)
public void testEnvExpansion() throws IOException {
  Path logPath = new Path("/nm/container/logs");
  String input =
      Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_HOME") + "/share/hadoop/common/lib/*"
          + ApplicationConstants.CLASS_PATH_SEPARATOR
          + Apps.crossPlatformify("HADOOP_LOG_HOME")
          + ApplicationConstants.LOG_DIR_EXPANSION_VAR;

  String res = ContainerLaunch.expandEnvironment(input, logPath);

  if (Shell.WINDOWS) {
    Assert.assertEquals("%HADOOP_HOME%/share/hadoop/common/*;"
        + "%HADOOP_HOME%/share/hadoop/common/lib/*;"
        + "%HADOOP_LOG_HOME%/nm/container/logs", res);
  } else {
    Assert.assertEquals("$HADOOP_HOME/share/hadoop/common/*:"
        + "$HADOOP_HOME/share/hadoop/common/lib/*:"
        + "$HADOOP_LOG_HOME/nm/container/logs", res);
  }
  System.out.println(res);
}
 

import org.apache.hadoop.yarn.util.Apps; //导入依赖的package包/类
/**
 * Get environment variables in a {@link java.util.Map} used when launching a Yarn container.
 *
 * @param yarnConfiguration a Hadoop {@link Configuration} object carrying Hadoop/Yarn configuration properties
 * @return a {@link java.util.Map} storing environment variables used when launching a Yarn container
 */
public static Map<String, String> getEnvironmentVariables(Configuration yarnConfiguration) {
  Map<String, String> environmentVariableMap = Maps.newHashMap();

  Apps.addToEnvironment(environmentVariableMap, ApplicationConstants.Environment.JAVA_HOME.key(),
      System.getenv(ApplicationConstants.Environment.JAVA_HOME.key()));

  // Add jars/files in the working directory of the ApplicationMaster to the CLASSPATH
  Apps.addToEnvironment(environmentVariableMap, ApplicationConstants.Environment.CLASSPATH.key(),
      ApplicationConstants.Environment.PWD.$());
  Apps.addToEnvironment(environmentVariableMap, ApplicationConstants.Environment.CLASSPATH.key(),
      ApplicationConstants.Environment.PWD.$() + File.separator + "*");

  String[] classpaths = yarnConfiguration.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
      YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH);
  if (classpaths != null) {
    for (String classpath : classpaths) {
      Apps.addToEnvironment(
          environmentVariableMap, ApplicationConstants.Environment.CLASSPATH.key(), classpath.trim());
    }
  }

  return environmentVariableMap;
}
 

import org.apache.hadoop.yarn.util.Apps; //导入依赖的package包/类
void setupAppClasspath(boolean miniClusterEnv, Configuration conf, Map<String, String> appMasterEnv) {
  if(miniClusterEnv) {
    String cps = System.getProperty("java.class.path") ;
    String[] cp = cps.split(":") ;
    for(String selCp : cp) {
      Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), selCp, ":");
    }
  } else {
    StringBuilder classPathEnv = new StringBuilder();
    classPathEnv.append(Environment.CLASSPATH.$()).append(File.pathSeparatorChar);
    classPathEnv.append("./*");

    String[] classpath = conf.getStrings(
        YarnConfiguration.YARN_APPLICATION_CLASSPATH,
        YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH
    ) ;
    for (String selClasspath : classpath) {
      classPathEnv.append(File.pathSeparatorChar);
      classPathEnv.append(selClasspath.trim());
    }
    appMasterEnv.put(Environment.CLASSPATH.name(), classPathEnv.toString());
    System.err.println("CLASSPATH: " + classPathEnv);
  }
  Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(), Environment.PWD.$() + File.separator + "*", ":");
}
 

import org.apache.hadoop.yarn.util.Apps; //导入依赖的package包/类
@SuppressWarnings("deprecation")
public static void setVMEnv(Map<String, String> environment, Configuration conf) {
  // Add the env variables passed by the user
  String setEnv = getChildEnv(conf);
  Apps.setEnvFromInputString(environment, setEnv);

  // Set logging level in the environment.
  // This is so that, if the child forks another "bin/hadoop" (common in
  // streaming) it will have the correct loglevel.
  environment.put("HADOOP_ROOT_LOGGER", getChildLogLevel(conf) + ",CLA");

  String hadoopClientOpts = System.getenv("HADOOP_CLIENT_OPTS");
  if (hadoopClientOpts == null) {
    hadoopClientOpts = "";
  } else {
    hadoopClientOpts = hadoopClientOpts + " ";
  }

  long logSize = 0;
  Vector<String> logProps = new Vector<String>(4);
  setupLog4jProperties(conf, logProps, logSize);
  Iterator<String> it = logProps.iterator();
  StringBuilder buffer = new StringBuilder();
  while (it.hasNext()) {
    buffer.append(" ").append(it.next());
  }
  hadoopClientOpts = hadoopClientOpts + buffer.toString();
  environment.put("HADOOP_CLIENT_OPTS", hadoopClientOpts);
}
 

import org.apache.hadoop.yarn.util.Apps; //导入依赖的package包/类
static ContainerLaunchContext createContainerLaunchContext(
    Map<ApplicationAccessType, String> applicationACLs, Configuration conf,
    PSAgentAttemptId attemptId, final ApplicationId appid, MasterService masterService,
    Credentials credentials) {

  synchronized (commonContainerSpecLock) {
    if (commonContainerSpec == null) {
      commonContainerSpec =
          createCommonContainerLaunchContext(masterService, applicationACLs, conf, appid,
              credentials);
    }
  }

  Map<String, String> env = commonContainerSpec.getEnvironment();
  Map<String, String> myEnv = new HashMap<String, String>(env.size());
  myEnv.putAll(env);

  Apps.addToEnvironment(myEnv, AngelEnvironment.PSAGENT_ID.name(),
      Integer.toString(attemptId.getPsAgentId().getIndex()));
  Apps.addToEnvironment(myEnv, AngelEnvironment.PSAGENT_ATTEMPT_ID.name(),
      Integer.toString(attemptId.getIndex()));

  //ParameterServerJVM.setVMEnv(myEnv, conf);

  // Set up the launch command
  List<String> commands = PSAgentAttemptJVM.getVMCommand(conf, appid, attemptId);

  // Duplicate the ByteBuffers for access by multiple containers.
  Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>();
  for (Entry<String, ByteBuffer> entry : commonContainerSpec.getServiceData().entrySet()) {
    myServiceData.put(entry.getKey(), entry.getValue().duplicate());
  }

  // Construct the actual Container
  ContainerLaunchContext container =
      ContainerLaunchContext.newInstance(commonContainerSpec.getLocalResources(), myEnv,
          commands, myServiceData, commonContainerSpec.getTokens().duplicate(), applicationACLs);

  return container;
}
 

import org.apache.hadoop.yarn.util.Apps; //导入依赖的package包/类
/**
 * Set environment variables of worker attempt process
 * @param environment environment variables of ps attempt process
 * @param conf application configuration
 */
public static void setVMEnv(Map<String, String> environment, Configuration conf) {
  // Add the env variables passed by the user
  String workerChildEnv = getChildEnv(conf);
  LOG.info("worker env=" + workerChildEnv);
  try{
    Apps.setEnvFromInputString(environment, workerChildEnv);
  } catch (Exception x) {
    LOG.error("set worker env failed.",  x);
  }
  
  // Set logging level in the environment.
  environment.put("HADOOP_ROOT_LOGGER", getChildLogLevel(conf) + ",CLA");

  String hadoopClientOpts = System.getenv("HADOOP_CLIENT_OPTS");
  if (hadoopClientOpts == null) {
    hadoopClientOpts = "";
  } else {
    hadoopClientOpts = hadoopClientOpts + " ";
  }

  long logSize = 0;
  Vector<String> logProps = new Vector<String>(4);
  setupLog4jProperties(conf, logProps, logSize);
  Iterator<String> it = logProps.iterator();
  StringBuilder buffer = new StringBuilder();
  while (it.hasNext()) {
    buffer.append(" ").append(it.next());
  }
  hadoopClientOpts = hadoopClientOpts + buffer.toString();
  environment.put("HADOOP_CLIENT_OPTS", hadoopClientOpts);
}
 

import org.apache.hadoop.yarn.util.Apps; //导入依赖的package包/类
@SuppressWarnings("deprecation")
public static void setClasspath(Map<String, String> environment, Configuration conf)
    throws IOException {
  String classpathEnvVar = Environment.CLASSPATH.name();
  Apps.addToEnvironment(environment, classpathEnvVar, Environment.PWD.$());
  Apps.addToEnvironment(environment, classpathEnvVar, Environment.PWD.$() + Path.SEPARATOR + "*");
  // a * in the classpath will only find a .jar, so we need to filter out
  // all .jars and add everything else
  addToClasspathIfNotJar(DistributedCache.getFileClassPaths(conf),
      DistributedCache.getCacheFiles(conf), conf, environment, classpathEnvVar);
  addToClasspathIfNotJar(DistributedCache.getArchiveClassPaths(conf),
      DistributedCache.getCacheArchives(conf), conf, environment, classpathEnvVar);
  
  AngelApps.setAngelFrameworkClasspath(environment, conf);
}
 

import org.apache.hadoop.yarn.util.Apps; //导入依赖的package包/类
/**
 * Set environment variables of ps attempt process
 * @param environment environment variables of ps attempt process
 * @param conf application configuration
 */
public static void setVMEnv(Map<String, String> environment, Configuration conf) {
  // Add the env variables passed by the user
  String setEnv = getChildEnv(conf);
  try{
    Apps.setEnvFromInputString(environment, setEnv);
  } catch (Exception x) {
    LOG.error("set ps env faile.", x);
  }    

  // Set logging level in the environment.
  environment.put("HADOOP_ROOT_LOGGER", getChildLogLevel(conf) + ",CLA");

  String hadoopClientOpts = System.getenv("HADOOP_CLIENT_OPTS");
  if (hadoopClientOpts == null) {
    hadoopClientOpts = "";
  } else {
    hadoopClientOpts = hadoopClientOpts + " ";
  }

  long logSize = 0;
  Vector<String> logProps = new Vector<String>(4);
  setupLog4jProperties(conf, logProps, logSize);
  Iterator<String> it = logProps.iterator();
  StringBuilder buffer = new StringBuilder();
  while (it.hasNext()) {
    buffer.append(" ").append(it.next());
  }
  hadoopClientOpts = hadoopClientOpts + buffer.toString();
  environment.put("HADOOP_CLIENT_OPTS", hadoopClientOpts);
}
 

import org.apache.hadoop.yarn.util.Apps; //导入依赖的package包/类
private void setupAppEnv(Map<String, String> appEnv, String exclasspath) {
	for (String c: conf.getStrings(
			YarnConfiguration.YARN_APPLICATION_CLASSPATH, 
			YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
		Apps.addToEnvironment(appEnv, Environment.CLASSPATH.name(), c.trim());
	}
	Apps.addToEnvironment(appEnv, Environment.CLASSPATH.name(),
			Environment.PWD.$() + File.separator + "*");
}
 

import org.apache.hadoop.yarn.util.Apps; //导入依赖的package包/类
public static void setEnvFromInputString(Map<String, String> env,
    String envString, Configuration conf) {
  String classPathSeparator =
      conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
        MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
          ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  Apps.setEnvFromInputString(env, envString, classPathSeparator);
}
 

import org.apache.hadoop.yarn.util.Apps; //导入依赖的package包/类
@Public
@Unstable
public static void addToEnvironment(Map<String, String> environment,
    String variable, String value, Configuration conf) {
  String classPathSeparator =
      conf.getBoolean(MRConfig.MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM,
        MRConfig.DEFAULT_MAPREDUCE_APP_SUBMISSION_CROSS_PLATFORM)
          ? ApplicationConstants.CLASS_PATH_SEPARATOR : File.pathSeparator;
  Apps.addToEnvironment(environment, variable, value, classPathSeparator);
}
 

import org.apache.hadoop.yarn.util.Apps; //导入依赖的package包/类
private void setupAppMasterEnv(YarnConfiguration conf, Map<String, String> appMasterEnv) {
    String[] defaultPaths = conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
                                            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH);
    for (String c : defaultPaths) {
        Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(),
                              c.trim(), File.pathSeparator);
    }
    Apps.addToEnvironment(appMasterEnv,
                          Environment.CLASSPATH.name(),
                          Environment.PWD.$() + File.separator + "*",
                          File.pathSeparator);
}
 

import org.apache.hadoop.yarn.util.Apps; //导入依赖的package包/类
/**
 * @param envs Environment variables.
 * @param conf Yarn configuration.
 */
private static void setupAppMasterEnv(Map<String, String> envs, YarnConfiguration conf) {
    for (String c : conf.getStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
        YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH))
        Apps.addToEnvironment(envs, Environment.CLASSPATH.name(),
                c.trim(), File.pathSeparator);

    Apps.addToEnvironment(envs,
            Environment.CLASSPATH.name(),
            Environment.PWD.$() + File.separator + "*",
            File.pathSeparator);
}
 

import org.apache.hadoop.yarn.util.Apps; //导入依赖的package包/类
@SuppressWarnings("deprecation")
private void setupAppMasterEnv(Map<String, String> appMasterEnv) {
    for (String c : conf.getStrings(
            YarnConfiguration.YARN_APPLICATION_CLASSPATH,
            YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) {
        Apps.addToEnvironment(appMasterEnv, Environment.CLASSPATH.name(),
                c.trim());
    }
    Apps.addToEnvironment(appMasterEnv,
            Environment.CLASSPATH.name(),
            Environment.PWD.$() + File.separator + "*");
}