Java LocalResource类代码示例

import org.apache.hadoop.yarn.api.records.LocalResource; //导入依赖的package包/类
public static ResourceLocalizationSpec newResourceLocalizationSpec(
    LocalResource rsrc, Path path) {
  URL local = ConverterUtils.getYarnUrlFromPath(path);
  ResourceLocalizationSpec resourceLocalizationSpec =
      Records.newRecord(ResourceLocalizationSpec.class);
  resourceLocalizationSpec.setDestinationDirectory(local);
  resourceLocalizationSpec.setResource(rsrc);
  return resourceLocalizationSpec;
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入依赖的package包/类
public void addResource(FileSystem fs, Configuration conf, Path destPath,
    Map<String, LocalResource> localResources, LocalResourceType resourceType, String link,
    Map<URI, FileStatus> statCache, boolean appMasterOnly) throws IOException {

  FileStatus destStatus = fs.getFileStatus(destPath);
  LocalResource amJarRsrc = Records.newRecord(LocalResource.class);
  amJarRsrc.setType(resourceType);

  LocalResourceVisibility visibility = getVisibility(conf, destPath.toUri(), statCache);
  amJarRsrc.setVisibility(visibility);
  amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(destPath));
  amJarRsrc.setTimestamp(destStatus.getModificationTime());
  amJarRsrc.setSize(destStatus.getLen());

  if (link == null || link.isEmpty())
    throw new IOException("You must specify a valid link name");

  localResources.put(link, amJarRsrc);
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入依赖的package包/类
@SuppressWarnings("deprecation")
public static void setupDistributedCache(Configuration conf,
    Map<String, LocalResource> localResources) throws IOException {

  // Cache archives
  parseDistributedCacheArtifacts(conf, localResources, LocalResourceType.ARCHIVE,
      DistributedCache.getCacheArchives(conf), DistributedCache.getArchiveTimestamps(conf),
      getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES),
      DistributedCache.getArchiveVisibilities(conf));

  // Cache files
  parseDistributedCacheArtifacts(conf, localResources, LocalResourceType.FILE,
      DistributedCache.getCacheFiles(conf), DistributedCache.getFileTimestamps(conf),
      getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES),
      DistributedCache.getFileVisibilities(conf));
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入依赖的package包/类
public boolean run() throws Exception {
  YarnClientApplication app = createApplication();
  ApplicationId appId = app.getNewApplicationResponse().getApplicationId();

  // Copy the application jar to the filesystem
  FileSystem fs = FileSystem.get(conf);
  String appIdStr = appId.toString();
  Path dstJarPath = Utils.copyLocalFileToDfs(fs, appIdStr, new Path(tfJar), Constants.TF_JAR_NAME);
  Path dstLibPath = Utils.copyLocalFileToDfs(fs, appIdStr, new Path(tfLib),
      Constants.TF_LIB_NAME);
  Map<String, Path> files = new HashMap<>();
  files.put(Constants.TF_JAR_NAME, dstJarPath);
  Map<String, LocalResource> localResources = Utils.makeLocalResources(fs, files);
  Map<String, String> javaEnv = Utils.setJavaEnv(conf);
  String command = makeAppMasterCommand(dstLibPath.toString(), dstJarPath.toString());
  LOG.info("Make ApplicationMaster command: " + command);
  ContainerLaunchContext launchContext = ContainerLaunchContext.newInstance(
      localResources, javaEnv, Lists.newArrayList(command), null, null, null);
  Resource resource = Resource.newInstance(amMemory, amVCores);
  submitApplication(app, appName, launchContext, resource, amQueue);
  return awaitApplication(appId);
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入依赖的package包/类
static LocalResource createJar(FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  LOG.info("Create jar file " + p);
  File jarFile = new File((files.makeQualified(p)).toUri());
  FileOutputStream stream = new FileOutputStream(jarFile);
  LOG.info("Create jar out stream ");
  JarOutputStream out = new JarOutputStream(stream, new Manifest());
  LOG.info("Done writing jar stream ");
  out.close();
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(p));
  FileStatus status = files.getFileStatus(p);
  ret.setSize(status.getLen());
  ret.setTimestamp(status.getModificationTime());
  ret.setType(LocalResourceType.PATTERN);
  ret.setVisibility(vis);
  ret.setPattern("classes/.*");
  return ret;
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入依赖的package包/类
static LocalResource createJarFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException,
    URISyntaxException {
  byte[] bytes = new byte[len];
  r.nextBytes(bytes);

  File archiveFile = new File(p.toUri().getPath() + ".jar");
  archiveFile.createNewFile();
  JarOutputStream out = new JarOutputStream(
      new FileOutputStream(archiveFile));
  out.putNextEntry(new JarEntry(p.getName()));
  out.write(bytes);
  out.closeEntry();
  out.close();

  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
      + ".jar")));
  ret.setSize(len);
  ret.setType(LocalResourceType.ARCHIVE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".jar"))
      .getModificationTime());
  return ret;
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入依赖的package包/类
static LocalResource createZipFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException,
    URISyntaxException {
  byte[] bytes = new byte[len];
  r.nextBytes(bytes);

  File archiveFile = new File(p.toUri().getPath() + ".ZIP");
  archiveFile.createNewFile();
  ZipOutputStream out = new ZipOutputStream(
      new FileOutputStream(archiveFile));
  out.putNextEntry(new ZipEntry(p.getName()));
  out.write(bytes);
  out.closeEntry();
  out.close();

  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
      + ".ZIP")));
  ret.setSize(len);
  ret.setType(LocalResourceType.ARCHIVE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".ZIP"))
      .getModificationTime());
  return ret;
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入依赖的package包/类
private Path getPathForLocalization(LocalResource rsrc) throws IOException,
    URISyntaxException {
  String user = context.getUser();
  ApplicationId appId =
      context.getContainerId().getApplicationAttemptId().getApplicationId();
  LocalResourceVisibility vis = rsrc.getVisibility();
  LocalResourcesTracker tracker =
      getLocalResourcesTracker(vis, user, appId);
  String cacheDirectory = null;
  if (vis == LocalResourceVisibility.PRIVATE) {// PRIVATE Only
    cacheDirectory = getUserFileCachePath(user);
  } else {// APPLICATION ONLY
    cacheDirectory = getAppFileCachePath(user, appId.toString());
  }
  Path dirPath =
      dirsHandler.getLocalPathForWrite(cacheDirectory,
        ContainerLocalizer.getEstimatedSize(rsrc), false);
  return tracker.getPathForLocalization(new LocalResourceRequest(rsrc),
      dirPath, delService);
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入依赖的package包/类
/**
 * @param resource the local resource that contains the original remote path
 * @param localPath the path in the local filesystem where the resource is
 * localized
 * @param fs the filesystem of the shared cache
 * @param localFs the local filesystem
 */
public SharedCacheUploader(LocalResource resource, Path localPath,
    String user, Configuration conf, SCMUploaderProtocol scmClient,
    FileSystem fs, FileSystem localFs) {
  this.resource = resource;
  this.localPath = localPath;
  this.user = user;
  this.conf = conf;
  this.scmClient = scmClient;
  this.fs = fs;
  this.sharedCacheRootDir =
      conf.get(YarnConfiguration.SHARED_CACHE_ROOT,
          YarnConfiguration.DEFAULT_SHARED_CACHE_ROOT);
  this.nestedLevel = SharedCacheUtil.getCacheDepth(conf);
  this.checksum = SharedCacheChecksumFactory.getChecksum(conf);
  this.localFs = localFs;
  this.recordFactory = RecordFactoryProvider.getRecordFactory(null);
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入依赖的package包/类
public ContainerLocalizer(FileContext lfs, String user, String appId,
    String localizerId, List<Path> localDirs,
    RecordFactory recordFactory) throws IOException {
  if (null == user) {
    throw new IOException("Cannot initialize for null user");
  }
  if (null == localizerId) {
    throw new IOException("Cannot initialize for null containerId");
  }
  this.lfs = lfs;
  this.user = user;
  this.appId = appId;
  this.localDirs = localDirs;
  this.localizerId = localizerId;
  this.recordFactory = recordFactory;
  this.conf = new Configuration();
  this.appCacheDirContextName = String.format(APPCACHE_CTXT_FMT, appId);
  this.pendingResources = new HashMap<LocalResource,Future<Path>>();
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入依赖的package包/类
static ResourceLocalizationSpec getMockRsrc(Random r,
    LocalResourceVisibility vis, Path p) {
  ResourceLocalizationSpec resourceLocalizationSpec =
    mock(ResourceLocalizationSpec.class);

  LocalResource rsrc = mock(LocalResource.class);
  String name = Long.toHexString(r.nextLong());
  URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
  when(uri.getScheme()).thenReturn("file");
  when(uri.getHost()).thenReturn(null);
  when(uri.getFile()).thenReturn("/local/" + vis + "/" + name);

  when(rsrc.getResource()).thenReturn(uri);
  when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L);
  when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L);
  when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
  when(rsrc.getVisibility()).thenReturn(vis);

  when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
  when(resourceLocalizationSpec.getDestinationDirectory()).
    thenReturn(ConverterUtils.getYarnUrlFromPath(p));
  return resourceLocalizationSpec;
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入依赖的package包/类
/**
 * If resource is public, verifyAccess should succeed
 */
@Test
public void testVerifyAccessPublicResource() throws Exception {
  Configuration conf = new Configuration();
  conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true);
  LocalResource resource = mock(LocalResource.class);
  // give public visibility
  when(resource.getVisibility()).thenReturn(LocalResourceVisibility.PUBLIC);
  Path localPath = mock(Path.class);
  when(localPath.getName()).thenReturn("foo.jar");
  String user = "joe";
  SCMUploaderProtocol scmClient = mock(SCMUploaderProtocol.class);
  FileSystem fs = mock(FileSystem.class);
  FileSystem localFs = FileSystem.getLocal(conf);
  SharedCacheUploader spied =
      createSpiedUploader(resource, localPath, user, conf, scmClient, fs,
          localFs);

  assertTrue(spied.verifyAccess());
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入依赖的package包/类
/**
 * If the localPath does not exists, getActualPath should get to one level
 * down
 */
@Test
public void testGetActualPath() throws Exception {
  Configuration conf = new Configuration();
  conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true);
  LocalResource resource = mock(LocalResource.class);
  // give public visibility
  when(resource.getVisibility()).thenReturn(LocalResourceVisibility.PUBLIC);
  Path localPath = new Path("foo.jar");
  String user = "joe";
  SCMUploaderProtocol scmClient = mock(SCMUploaderProtocol.class);
  FileSystem fs = mock(FileSystem.class);
  FileSystem localFs = mock(FileSystem.class);
  // stub it to return a status that indicates a directory
  FileStatus status = mock(FileStatus.class);
  when(status.isDirectory()).thenReturn(true);
  when(localFs.getFileStatus(localPath)).thenReturn(status);
  SharedCacheUploader spied =
      createSpiedUploader(resource, localPath, user, conf, scmClient, fs,
          localFs);

  Path actualPath = spied.getActualPath();
  assertEquals(actualPath.getName(), localPath.getName());
  assertEquals(actualPath.getParent().getName(), localPath.getName());
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入依赖的package包/类
public Map<Path, List<String>> doLocalizeResources(
    boolean checkLocalizingState, int skipRsrcCount)
    throws URISyntaxException {
  Path cache = new Path("file:///cache");
  Map<Path, List<String>> localPaths =
      new HashMap<Path, List<String>>();
  int counter = 0;
  for (Entry<String, LocalResource> rsrc : localResources.entrySet()) {
    if (counter++ < skipRsrcCount) {
      continue;
    }
    if (checkLocalizingState) {
      assertEquals(ContainerState.LOCALIZING, c.getContainerState());
    }
    LocalResourceRequest req = new LocalResourceRequest(rsrc.getValue());
    Path p = new Path(cache, rsrc.getKey());
    localPaths.put(p, Arrays.asList(rsrc.getKey()));
    // rsrc copied to p
    c.handle(new ContainerResourceLocalizedEvent(c.getContainerId(),
             req, p));
  }
  drainDispatcherEvents();
  return localPaths;
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入依赖的package包/类
static LocalizerHeartbeatResponse createLocalizerHeartbeatResponse() 
    throws URISyntaxException {
  LocalizerHeartbeatResponse ret =
    recordFactory.newRecordInstance(LocalizerHeartbeatResponse.class);
  assertTrue(ret instanceof LocalizerHeartbeatResponsePBImpl);
  ret.setLocalizerAction(LocalizerAction.LIVE);
  LocalResource rsrc = createResource();
  ArrayList<ResourceLocalizationSpec> rsrcs =
    new ArrayList<ResourceLocalizationSpec>();
  ResourceLocalizationSpec resource =
    recordFactory.newRecordInstance(ResourceLocalizationSpec.class);
  resource.setResource(rsrc);
  resource.setDestinationDirectory(ConverterUtils
    .getYarnUrlFromPath(new Path("/tmp" + System.currentTimeMillis())));
  rsrcs.add(resource);
  ret.setResourceSpecs(rsrcs);
  System.out.println(resource);
  return ret;
}