Java Shell.appendScriptExtension方法代码示例

import org.apache.hadoop.util.Shell; //导入方法依赖的package包/类
/**
 * Creates a script to run a container that will run forever unless
 * stopped by external means.
 */
private static File createUnhaltingScriptFile(ContainerId cId,
    File scriptFileDir, File processStartFile) throws IOException {
  File scriptFile = Shell.appendScriptExtension(scriptFileDir, "scriptFile");
  PrintWriter fileWriter = new PrintWriter(scriptFile);
  if (Shell.WINDOWS) {
    fileWriter.println("@echo \"Running testscript for delayed kill\"");
    fileWriter.println("@echo \"Writing pid to start file\"");
    fileWriter.println("@echo " + cId + ">> " + processStartFile);
    fileWriter.println("@pause");
  } else {
    fileWriter.write("#!/bin/bash\n\n");
    fileWriter.write("echo \"Running testscript for delayed kill\"\n");
    fileWriter.write("hello=\"Got SIGTERM\"\n");
    fileWriter.write("umask 0\n");
    fileWriter.write("trap \"echo $hello >> " + processStartFile +
      "\" SIGTERM\n");
    fileWriter.write("echo \"Writing pid to start file\"\n");
    fileWriter.write("echo $$ >> " + processStartFile + "\n");
    fileWriter.write("while true; do\ndate >> /dev/null;\n done\n");
  }

  fileWriter.close();
  return scriptFile;
}
 

import org.apache.hadoop.util.Shell; //导入方法依赖的package包/类
public UnixLocalWrapperScriptBuilder(Path containerWorkDir, String dockerCommand, String dockerPidScript) {
  super(containerWorkDir);
  this.dockerCommand = dockerCommand;
  this.dockerPidScript = dockerPidScript;
  this.sessionScriptPath = new Path(containerWorkDir,
    Shell.appendScriptExtension(DOCKER_CONTAINER_EXECUTOR_SESSION_SCRIPT));
}
 

import org.apache.hadoop.util.Shell; //导入方法依赖的package包/类
/**
 * Test that script exists with non-zero exit code when command fails.
 * @throws IOException
 */
@Test (timeout = 10000)
public void testShellScriptBuilderNonZeroExitCode() throws IOException {
  ShellScriptBuilder builder = ShellScriptBuilder.create();
  builder.command(Arrays.asList(new String[] {"unknownCommand"}));
  File shellFile = Shell.appendScriptExtension(tmpDir, "testShellScriptBuilderError");
  PrintStream writer = new PrintStream(new FileOutputStream(shellFile));
  builder.write(writer);
  writer.close();
  try {
    FileUtil.setExecutable(shellFile, true);

    Shell.ShellCommandExecutor shexc = new Shell.ShellCommandExecutor(
        new String[]{shellFile.getAbsolutePath()}, tmpDir);
    try {
      shexc.execute();
      fail("builder shell command was expected to throw");
    }
    catch(IOException e) {
      // expected
      System.out.println("Received an expected exception: " + e.getMessage());
    }
  }
  finally {
    FileUtil.fullyDelete(shellFile);
  }
}
 

import org.apache.hadoop.util.Shell; //导入方法依赖的package包/类
protected LocalWrapperScriptBuilder(Path containerWorkDir) {
  this.wrapperScriptPath = new Path(containerWorkDir,
      Shell.appendScriptExtension(DOCKER_CONTAINER_EXECUTOR_SCRIPT));
}
 

import org.apache.hadoop.util.Shell; //导入方法依赖的package包/类
protected LocalWrapperScriptBuilder(Path containerWorkDir) {
  this.wrapperScriptPath = new Path(containerWorkDir,
    Shell.appendScriptExtension("default_container_executor"));
}
 

import org.apache.hadoop.util.Shell; //导入方法依赖的package包/类
public UnixLocalWrapperScriptBuilder(Path containerWorkDir) {
  super(containerWorkDir);
  this.sessionScriptPath = new Path(containerWorkDir,
      Shell.appendScriptExtension("default_container_executor_session"));
}
 

import org.apache.hadoop.util.Shell; //导入方法依赖的package包/类
@Test (timeout = 20000)
public void testInvalidEnvSyntaxDiagnostics() throws IOException  {

  File shellFile = null;
  try {
    shellFile = Shell.appendScriptExtension(tmpDir, "hello");
    Map<Path, List<String>> resources =
        new HashMap<Path, List<String>>();
    FileOutputStream fos = new FileOutputStream(shellFile);
    FileUtil.setExecutable(shellFile, true);

    Map<String, String> env = new HashMap<String, String>();
    // invalid env
    env.put(
        "APPLICATION_WORKFLOW_CONTEXT", "{\"workflowId\":\"609f91c5cd83\"," +
        "\"workflowName\":\"\n\ninsert table " +
        "\npartition (cd_education_status)\nselect cd_demo_sk, cd_gender, " );
    List<String> commands = new ArrayList<String>();
    new DefaultContainerExecutor().writeLaunchEnv(fos, env, resources, commands);
    fos.flush();
    fos.close();

    // It is supposed that LANG is set as C.
    Map<String, String> cmdEnv = new HashMap<String, String>();
    cmdEnv.put("LANG", "C");
    Shell.ShellCommandExecutor shexc
    = new Shell.ShellCommandExecutor(new String[]{shellFile.getAbsolutePath()},
      tmpDir, cmdEnv);
    String diagnostics = null;
    try {
      shexc.execute();
      Assert.fail("Should catch exception");
    } catch(ExitCodeException e){
      diagnostics = e.getMessage();
    }
    Assert.assertTrue(diagnostics.contains(Shell.WINDOWS ?
        "is not recognized as an internal or external command" :
        "command not found"));
    Assert.assertTrue(shexc.getExitCode() != 0);
  }
  finally {
    // cleanup
    if (shellFile != null
        && shellFile.exists()) {
      shellFile.delete();
    }
  }
}
 

import org.apache.hadoop.util.Shell; //导入方法依赖的package包/类
@Test (timeout = 20000)
public void testContainerLaunchStdoutAndStderrDiagnostics() throws IOException {

  File shellFile = null;
  try {
    shellFile = Shell.appendScriptExtension(tmpDir, "hello");
    // echo "hello" to stdout and "error" to stderr and exit code with 2;
    String command = Shell.WINDOWS ?
        "@echo \"hello\" & @echo \"error\" 1>&2 & exit /b 2" :
        "echo \"hello\"; echo \"error\" 1>&2; exit 2;";
    PrintWriter writer = new PrintWriter(new FileOutputStream(shellFile));
    FileUtil.setExecutable(shellFile, true);
    writer.println(command);
    writer.close();
    Map<Path, List<String>> resources =
        new HashMap<Path, List<String>>();
    FileOutputStream fos = new FileOutputStream(shellFile, true);

    Map<String, String> env = new HashMap<String, String>();
    List<String> commands = new ArrayList<String>();
    commands.add(command);
    ContainerExecutor exec = new DefaultContainerExecutor();
    exec.writeLaunchEnv(fos, env, resources, commands);
    fos.flush();
    fos.close();

    Shell.ShellCommandExecutor shexc
    = new Shell.ShellCommandExecutor(new String[]{shellFile.getAbsolutePath()}, tmpDir);
    String diagnostics = null;
    try {
      shexc.execute();
      Assert.fail("Should catch exception");
    } catch(ExitCodeException e){
      diagnostics = e.getMessage();
    }
    // test stderr
    Assert.assertTrue(diagnostics.contains("error"));
    // test stdout
    Assert.assertTrue(shexc.getOutput().contains("hello"));
    Assert.assertTrue(shexc.getExitCode() == 2);
  }
  finally {
    // cleanup
    if (shellFile != null
        && shellFile.exists()) {
      shellFile.delete();
    }
  }
}
 

import org.apache.hadoop.util.Shell; //导入方法依赖的package包/类
private void testContainerLaunchAndExit(int exitCode) throws IOException,
    InterruptedException, YarnException {

 File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
 PrintWriter fileWriter = new PrintWriter(scriptFile);
 File processStartFile =
	  new File(tmpDir, "start_file.txt").getAbsoluteFile();

 // ////// Construct the Container-id
 ContainerId cId = createContainerId(0);

 if (Shell.WINDOWS) {
   fileWriter.println("@echo Hello World!> " + processStartFile);
   fileWriter.println("@echo " + cId + ">> " + processStartFile);
   if (exitCode != 0) {
     fileWriter.println("@exit " + exitCode);
   }
 } else {
   fileWriter.write("\numask 0"); // So that start file is readable by the test
   fileWriter.write("\necho Hello World! > " + processStartFile);
   fileWriter.write("\necho $$ >> " + processStartFile); 
   // Have script throw an exit code at the end
   if (exitCode != 0) {
     fileWriter.write("\nexit "+exitCode);
   }
 }
 
 fileWriter.close();

 ContainerLaunchContext containerLaunchContext = 
	  recordFactory.newRecordInstance(ContainerLaunchContext.class);

 URL resource_alpha =
	  ConverterUtils.getYarnUrlFromPath(localFS
			  .makeQualified(new Path(scriptFile.getAbsolutePath())));
 LocalResource rsrc_alpha =
	  recordFactory.newRecordInstance(LocalResource.class);
 rsrc_alpha.setResource(resource_alpha);
 rsrc_alpha.setSize(-1);
 rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
 rsrc_alpha.setType(LocalResourceType.FILE);
 rsrc_alpha.setTimestamp(scriptFile.lastModified());
 String destinationFile = "dest_file";
 Map<String, LocalResource> localResources = 
	  new HashMap<String, LocalResource>();
 localResources.put(destinationFile, rsrc_alpha);
 containerLaunchContext.setLocalResources(localResources);
 List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
 containerLaunchContext.setCommands(commands);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

 BaseContainerManagerTest.waitForContainerState(containerManager, cId,
	  ContainerState.COMPLETE);

  List<ContainerId> containerIds = new ArrayList<ContainerId>();
  containerIds.add(cId);
  GetContainerStatusesRequest gcsRequest =
      GetContainerStatusesRequest.newInstance(containerIds);
 ContainerStatus containerStatus = 
	  containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);

 // Verify exit status matches exit state of script
 Assert.assertEquals(exitCode,
	  containerStatus.getExitStatus());	    
}
 

import org.apache.hadoop.util.Shell; //导入方法依赖的package包/类
@Test
public void testContainerLaunch() throws IOException {
  String appSubmitter = "nobody";
  String appId = "APP_ID";
  String containerId = "CONTAINER_ID";
  String testImage = "\"sequenceiq/hadoop-docker:2.4.1\"";

  Container container = mock(Container.class, RETURNS_DEEP_STUBS);
  ContainerId cId = mock(ContainerId.class, RETURNS_DEEP_STUBS);
  ContainerLaunchContext context = mock(ContainerLaunchContext.class);
  HashMap<String, String> env = new HashMap<String,String>();

  when(container.getContainerId()).thenReturn(cId);
  when(container.getLaunchContext()).thenReturn(context);
  when(cId.getApplicationAttemptId().getApplicationId().toString()).thenReturn(appId);
  when(cId.toString()).thenReturn(containerId);

  when(context.getEnvironment()).thenReturn(env);
  env.put(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, testImage);
  Path scriptPath = new Path("file:///bin/echo");
  Path tokensPath = new Path("file:///dev/null");

  Path pidFile = new Path(workDir, "pid");

  dockerContainerExecutor.activateContainer(cId, pidFile);
  int ret = dockerContainerExecutor.launchContainer(
      new ContainerStartContext.Builder()
          .setContainer(container)
          .setNmPrivateContainerScriptPath(scriptPath)
          .setNmPrivateTokensPath(tokensPath)
          .setUser(appSubmitter)
          .setAppId(appId)
          .setContainerWorkDir(workDir)
          .setLocalDirs(dirsHandler.getLocalDirs())
          .setLogDirs(dirsHandler.getLogDirs())
          .build());
  assertEquals(0, ret);
  //get the script
  Path sessionScriptPath = new Path(workDir,
      Shell.appendScriptExtension(
          DockerContainerExecutor.DOCKER_CONTAINER_EXECUTOR_SESSION_SCRIPT));
  LineNumberReader lnr = new LineNumberReader(new FileReader(sessionScriptPath.toString()));
  boolean cmdFound = false;
  List<String> localDirs = dirsToMount(dirsHandler.getLocalDirs());
  List<String> logDirs = dirsToMount(dirsHandler.getLogDirs());
  List<String> workDirMount = dirsToMount(Collections.singletonList(workDir.toUri().getPath()));
  List<String> expectedCommands =  new ArrayList<String>(
      Arrays.asList(DOCKER_LAUNCH_COMMAND, "run", "--rm", "--net=host",  "--name", containerId));
  expectedCommands.addAll(localDirs);
  expectedCommands.addAll(logDirs);
  expectedCommands.addAll(workDirMount);
  String shellScript =  workDir + "/launch_container.sh";

  expectedCommands.addAll(Arrays.asList(testImage.replaceAll("['\"]", ""), "bash","\"" + shellScript + "\""));

  String expectedPidString = "echo `/bin/true inspect --format {{.State.Pid}} " + containerId+"` > "+ pidFile.toString() + ".tmp";
  boolean pidSetterFound = false;
  while(lnr.ready()){
    String line = lnr.readLine();
    LOG.debug("line: " + line);
    if (line.startsWith(DOCKER_LAUNCH_COMMAND)){
      List<String> command = new ArrayList<String>();
      for( String s :line.split("\\s+")){
        command.add(s.trim());
      }

      assertEquals(expectedCommands, command);
      cmdFound = true;
    } else if (line.startsWith("echo")) {
      assertEquals(expectedPidString, line);
      pidSetterFound = true;
    }

  }
  assertTrue(cmdFound);
  assertTrue(pidSetterFound);
}