Java URL.fromPath方法代码示例

import org.apache.hadoop.yarn.api.records.URL; //导入方法依赖的package包/类
@Test
public void testConvertUrlWithNoPort() throws URISyntaxException {
  Path expectedPath = new Path("hdfs://foo.com");
  URL url = URL.fromPath(expectedPath);
  Path actualPath = url.toPath();
  assertEquals(expectedPath, actualPath);
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入方法依赖的package包/类
@Test
public void testConvertUrlWithUserinfo() throws URISyntaxException {
  Path expectedPath = new Path("foo://username:[email protected]:8042");
  URL url = URL.fromPath(expectedPath);
  Path actualPath = url.toPath();
  assertEquals(expectedPath, actualPath);
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入方法依赖的package包/类
public static ResourceLocalizationSpec newResourceLocalizationSpec(
    LocalResource rsrc, Path path) {
  URL local = URL.fromPath(path);
  ResourceLocalizationSpec resourceLocalizationSpec =
      Records.newRecord(ResourceLocalizationSpec.class);
  resourceLocalizationSpec.setDestinationDirectory(local);
  resourceLocalizationSpec.setResource(rsrc);
  return resourceLocalizationSpec;
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入方法依赖的package包/类
/**
 * Create a {@link LocalResource} record with all the given parameters.
 */
private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = URL.fromPath(fc.resolvePath(fstat.getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
    resourceSize, resourceModificationTime);
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入方法依赖的package包/类
@Public
@Deprecated
public static URL getYarnUrlFromPath(Path path) {
  return URL.fromPath(path);
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入方法依赖的package包/类
@Override
public URL getResource() {
  return URL.fromPath(loc);
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入方法依赖的package包/类
@Test
public void testLogAggregationForRealContainerLaunch() throws IOException,
    InterruptedException, YarnException {

  this.containerManager.start();


  File scriptFile = new File(tmpDir, "scriptFile.sh");
  PrintWriter fileWriter = new PrintWriter(scriptFile);
  File userFold = new File(localLogDir, userFolder);
  fileWriter.write("\necho Hello World! Stdout! > "
      + new File(userFold, "stdout"));
  fileWriter.write("\necho Hello World! Stderr! > "
      + new File(userFold, "stderr"));
  fileWriter.write("\necho Hello World! Syslog! > "
      + new File(userFold, "syslog"));
  fileWriter.close();

  ContainerLaunchContext containerLaunchContext =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  // ////// Construct the Container-id
  ApplicationId appId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId appAttemptId =
      BuilderUtils.newApplicationAttemptId(appId, 1);
  ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0);

  URL resource_alpha =
      URL.fromPath(localFS
          .makeQualified(new Path(scriptFile.getAbsolutePath())));
  LocalResource rsrc_alpha =
      recordFactory.newRecordInstance(LocalResource.class);
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(scriptFile.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources = 
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);
  List<String> commands = new ArrayList<String>();
  commands.add("/bin/bash");
  commands.add(scriptFile.getAbsolutePath());
  containerLaunchContext.setCommands(commands);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(containerLaunchContext,
        TestContainerManager.createContainerToken(
          cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
          context.getContainerTokenSecretManager(), userFolder));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  this.containerManager.startContainers(allRequests);
  
  BaseContainerManagerTest.waitForContainerState(this.containerManager,
      cId, ContainerState.COMPLETE);

  this.containerManager.handle(new CMgrCompletedAppsEvent(Arrays
      .asList(appId), CMgrCompletedAppsEvent.Reason.ON_SHUTDOWN));
  this.containerManager.stop();
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入方法依赖的package包/类
@Test
public void testContainerSetup() throws Exception {

  containerManager.start();

  // ////// Create the resources for the container
  File dir = new File(tmpDir, "dir");
  dir.mkdirs();
  File file = new File(dir, "file");
  PrintWriter fileWriter = new PrintWriter(file);
  fileWriter.write("Hello World!");
  fileWriter.close();

  // ////// Construct the Container-id
  ContainerId cId = createContainerId(0);

  // ////// Construct the container-spec.
  ContainerLaunchContext containerLaunchContext = 
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  URL resource_alpha =
      URL.fromPath(localFS
          .makeQualified(new Path(file.getAbsolutePath())));
  LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);    
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(file.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources = 
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager(), userFolder));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

  BaseContainerManagerTest.waitForContainerState(containerManager, cId,
      ContainerState.COMPLETE);

  // Now ascertain that the resources are localised correctly.
  ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
  String appIDStr = appId.toString();
  String containerIDStr = cId.toString();
  File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
  File userDir = new File(userCacheDir, userFolder);
  File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
  File appDir = new File(appCache, appIDStr);
  File containerDir = new File(appDir, containerIDStr);
  File targetFile = new File(containerDir, destinationFile);
  File sysDir =
      new File(localDir,
          ResourceLocalizationService.NM_PRIVATE_DIR);
  File appSysDir = new File(sysDir, appIDStr);
  File containerSysDir = new File(appSysDir, containerIDStr);

  for (File f : new File[] { localDir, sysDir, userCacheDir, appDir,
      appSysDir,
      containerDir, containerSysDir }) {
    Assert.assertTrue(f.getAbsolutePath() + " doesn't exist!!", f.exists());
    Assert.assertTrue(f.getAbsolutePath() + " is not a directory!!",
        f.isDirectory());
  }
  Assert.assertTrue(targetFile.getAbsolutePath() + " doesn't exist!!",
      targetFile.exists());

  // Now verify the contents of the file
  BufferedReader reader = new BufferedReader(new FileReader(targetFile));
  Assert.assertEquals("Hello World!", reader.readLine());
  Assert.assertEquals(null, reader.readLine());
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入方法依赖的package包/类
private void testContainerLaunchAndExit(int exitCode) throws IOException,
    InterruptedException, YarnException {

 File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
 PrintWriter fileWriter = new PrintWriter(scriptFile);
 File processStartFile =
	  new File(tmpDir, "start_file.txt").getAbsoluteFile();

 // ////// Construct the Container-id
 ContainerId cId = createContainerId(0);

 if (Shell.WINDOWS) {
   fileWriter.println("@echo Hello World!> " + processStartFile);
   fileWriter.println("@echo " + cId + ">> " + processStartFile);
   if (exitCode != 0) {
     fileWriter.println("@exit " + exitCode);
   }
 } else {
   fileWriter.write("\numask 0"); // So that start file is readable by the test
   fileWriter.write("\necho Hello World! > " + processStartFile);
   fileWriter.write("\necho $$ >> " + processStartFile); 
   // Have script throw an exit code at the end
   if (exitCode != 0) {
     fileWriter.write("\nexit "+exitCode);
   }
 }
 
 fileWriter.close();

 ContainerLaunchContext containerLaunchContext = 
	  recordFactory.newRecordInstance(ContainerLaunchContext.class);

 URL resource_alpha =
	  URL.fromPath(localFS
			  .makeQualified(new Path(scriptFile.getAbsolutePath())));
 LocalResource rsrc_alpha =
	  recordFactory.newRecordInstance(LocalResource.class);
 rsrc_alpha.setResource(resource_alpha);
 rsrc_alpha.setSize(-1);
 rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
 rsrc_alpha.setType(LocalResourceType.FILE);
 rsrc_alpha.setTimestamp(scriptFile.lastModified());
 String destinationFile = "dest_file";
 Map<String, LocalResource> localResources = 
	  new HashMap<String, LocalResource>();
 localResources.put(destinationFile, rsrc_alpha);
 containerLaunchContext.setLocalResources(localResources);
 List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
 containerLaunchContext.setCommands(commands);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager(), userFolder));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

 BaseContainerManagerTest.waitForContainerState(containerManager, cId,
	  ContainerState.COMPLETE);

  List<ContainerId> containerIds = new ArrayList<ContainerId>();
  containerIds.add(cId);
  GetContainerStatusesRequest gcsRequest =
      GetContainerStatusesRequest.newInstance(containerIds);
 ContainerStatus containerStatus = 
	  containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);

 // Verify exit status matches exit state of script
 Assert.assertEquals(exitCode,
	  containerStatus.getExitStatus());	    
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入方法依赖的package包/类
private void testContainerLaunchAndSignal(SignalContainerCommand command)
    throws IOException, InterruptedException, YarnException {

  Signal signal = ContainerLaunch.translateCommandToSignal(command);
  containerManager.start();

  File scriptFile = new File(tmpDir, "scriptFile.sh");
  PrintWriter fileWriter = new PrintWriter(scriptFile);
  File processStartFile =
      new File(tmpDir, "start_file.txt").getAbsoluteFile();
  fileWriter.write("\numask 0"); // So that start file is readable by the test
  fileWriter.write("\necho Hello World! > " + processStartFile);
  fileWriter.write("\necho $$ >> " + processStartFile);
  fileWriter.write("\nexec sleep 1000s");
  fileWriter.close();

  ContainerLaunchContext containerLaunchContext =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);

  // ////// Construct the Container-id
  ContainerId cId = createContainerId(0);

  URL resource_alpha =
      URL.fromPath(localFS
          .makeQualified(new Path(scriptFile.getAbsolutePath())));
  LocalResource rsrc_alpha =
      recordFactory.newRecordInstance(LocalResource.class);
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(scriptFile.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources =
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);
  List<String> commands = new ArrayList<String>();
  commands.add("/bin/bash");
  commands.add(scriptFile.getAbsolutePath());
  containerLaunchContext.setCommands(commands);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
          containerLaunchContext,
          createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager(), userFolder));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

  int timeoutSecs = 0;
  while (!processStartFile.exists() && timeoutSecs++ < 20) {
    Thread.sleep(1000);
    LOG.info("Waiting for process start-file to be created");
  }
  Assert.assertTrue("ProcessStartFile doesn't exist!",
      processStartFile.exists());

  // Simulate NodeStatusUpdaterImpl sending CMgrSignalContainersEvent
  SignalContainerRequest signalReq =
      SignalContainerRequest.newInstance(cId, command);
  List<SignalContainerRequest> reqs = new ArrayList<SignalContainerRequest>();
  reqs.add(signalReq);
  containerManager.handle(new CMgrSignalContainersEvent(reqs));

  final ArgumentCaptor<ContainerSignalContext> signalContextCaptor =
      ArgumentCaptor.forClass(ContainerSignalContext.class);
  if (signal.equals(Signal.NULL)) {
    verify(exec, never()).signalContainer(signalContextCaptor.capture());
  } else {
    verify(exec, timeout(10000).atLeastOnce()).signalContainer(signalContextCaptor.capture());
    ContainerSignalContext signalContext = signalContextCaptor.getAllValues().get(0);
    Assert.assertEquals(cId, signalContext.getContainer().getContainerId());
    Assert.assertEquals(signal, signalContext.getSignal());
  }
}
 

import org.apache.hadoop.yarn.api.records.URL; //导入方法依赖的package包/类
public void startContainer()
    throws IOException, InterruptedException, YarnException {
  LOG.info("Start a container and wait until it is in RUNNING state");
  File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
  PrintWriter fileWriter = new PrintWriter(scriptFile);
  if (Shell.WINDOWS) {
    fileWriter.println("@ping -n 100 127.0.0.1 >nul");
  } else {
    fileWriter.write("\numask 0");
    fileWriter.write("\nexec sleep 100");
  }
  fileWriter.close();
  ContainerLaunchContext containerLaunchContext =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  URL resource_alpha =
      URL.fromPath(localFS
          .makeQualified(new Path(scriptFile.getAbsolutePath())));
  LocalResource rsrc_alpha =
      recordFactory.newRecordInstance(LocalResource.class);
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(scriptFile.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources =
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);
  List<String> commands =
      Arrays.asList(Shell.getRunScriptCommand(scriptFile));
  containerLaunchContext.setCommands(commands);
  Resource resource = Resource.newInstance(1024, 1);
  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
          containerLaunchContext,
          getContainerToken(resource));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  getContainerManager().startContainers(allRequests);
  // Make sure the container reaches RUNNING state
  ContainerId cId = TestContainerManager.createContainerId(0);
  BaseContainerManagerTest.waitForNMContainerState(
      getContainerManager(), cId,
      org.apache.hadoop.yarn.server.nodemanager.
          containermanager.container.ContainerState.RUNNING);
}