Java LocalResource.newInstance方法代码示例

import org.apache.hadoop.yarn.api.records.LocalResource; //导入方法依赖的package包/类
private void setUpLocalResources(ContainerLauncherEvent event) {
  String resourceFileName = event.getResourceFileName();
  String resourcePath = event.getResourceFilePath();
  if (resourcePath != "") {
    FileSystem fs = null;
    try {
      fs = FileSystem.get(new YarnConfiguration());
      Path dst = new Path(fs.getHomeDirectory(), resourcePath);
      boolean exists = fs.exists(dst);
      if (exists) {
        FileStatus scFileStatus = fs.getFileStatus(dst);
        LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()),
          LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(),
          scFileStatus.getModificationTime());
        localResources.put(resourceFileName, scRsrc);
      }
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入方法依赖的package包/类
private void addToLocalResources(FileSystem fs, String fileSrcPath, String fileDstPath, String appId, Map<String, LocalResource> localResources, String resources) throws IOException {
  String suffix = yacopConfig.getName() + "/" + appId + "/" + fileDstPath;
  Path dst = new Path(fs.getHomeDirectory(), suffix);
  if (fileSrcPath == null) {
    FSDataOutputStream ostream = null;
    try {
      ostream = FileSystem.create(fs, dst, new FsPermission((short) 0710));
      ostream.writeUTF(resources);
    } finally {
      IOUtils.closeQuietly(ostream);
    }
  } else {
    fs.copyFromLocalFile(new Path(fileSrcPath), dst);
  }
  FileStatus scFileStatus = fs.getFileStatus(dst);
  LocalResource scRsrc = LocalResource.newInstance(ConverterUtils.getYarnUrlFromURI(dst.toUri()), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, scFileStatus.getLen(), scFileStatus.getModificationTime());
  localResources.put(fileDstPath, scRsrc);
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入方法依赖的package包/类
private LocalResource toLocalResource(Path path, LocalResourceVisibility visibility) throws IOException {
  FileSystem fs = path.getFileSystem(clusterConf.conf());
  FileStatus stat = fs.getFileStatus(path);
  return LocalResource.newInstance(
          ConverterUtils.getYarnUrlFromPath(path),
          LocalResourceType.FILE,
          visibility,
          stat.getLen(), stat.getModificationTime()
  );
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入方法依赖的package包/类
private static void addToLocalResources(FileSystem fs, String key, Path dst,
    Map<String, LocalResource> localResources) throws IOException {
  FileStatus scFileStatus = fs.getFileStatus(dst);
  LocalResource resource =
      LocalResource.newInstance(
          URL.fromURI(dst.toUri()),
          LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
          scFileStatus.getLen(), scFileStatus.getModificationTime());
  localResources.put(key, resource);
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入方法依赖的package包/类
private void addToLocalResources(FileSystem fs, String fileSrcPath,
    String fileDstPath, String appId, Map<String, LocalResource> localResources,
    String resources) throws IOException {
  String suffix =
      appName + "/" + appId + "/" + fileDstPath;
  Path dst =
      new Path(fs.getHomeDirectory(), suffix);
  if (fileSrcPath == null) {
    FSDataOutputStream ostream = null;
    try {
      ostream = FileSystem
          .create(fs, dst, new FsPermission((short) 0710));
      ostream.writeUTF(resources);
    } finally {
      IOUtils.closeQuietly(ostream);
    }
  } else {
    fs.copyFromLocalFile(new Path(fileSrcPath), dst);
  }
  FileStatus scFileStatus = fs.getFileStatus(dst);
  LocalResource scRsrc =
      LocalResource.newInstance(
          ConverterUtils.getYarnUrlFromURI(dst.toUri()),
          LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
          scFileStatus.getLen(), scFileStatus.getModificationTime());
  localResources.put(fileDstPath, scRsrc);
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入方法依赖的package包/类
/**
 * Create a {@link LocalResource} record with all the given parameters.
 */
private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
    resourceSize, resourceModificationTime);
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入方法依赖的package包/类
/**
 * @return {@link Path} absolute path for localization which includes local
 *         directory path and the relative hierarchical path (if use local
 *         cache directory manager is enabled)
 * 
 * @param {@link LocalResourceRequest} Resource localization request to
 *        localize the resource.
 * @param {@link Path} local directory path
 */
@Override
public Path
    getPathForLocalization(LocalResourceRequest req, Path localDirPath) {
  Path rPath = localDirPath;
  if (useLocalCacheDirectoryManager && localDirPath != null) {

    if (!directoryManagers.containsKey(localDirPath)) {
      directoryManagers.putIfAbsent(localDirPath,
        new LocalCacheDirectoryManager(conf));
    }
    LocalCacheDirectoryManager dir = directoryManagers.get(localDirPath);

    rPath = localDirPath;
    String hierarchicalPath = dir.getRelativePathForLocalization();
    // For most of the scenarios we will get root path only which
    // is an empty string
    if (!hierarchicalPath.isEmpty()) {
      rPath = new Path(localDirPath, hierarchicalPath);
    }
    inProgressLocalResourcesMap.put(req, rPath);
  }

  rPath = new Path(rPath,
      Long.toString(uniqueNumberGenerator.incrementAndGet()));
  Path localPath = new Path(rPath, req.getPath().getName());
  LocalizedResource rsrc = localrsrc.get(req);
  rsrc.setLocalPath(localPath);
  LocalResource lr = LocalResource.newInstance(req.getResource(),
      req.getType(), req.getVisibility(), req.getSize(),
      req.getTimestamp());
  try {
    stateStore.startResourceLocalization(user, appId,
        ((LocalResourcePBImpl) lr).getProto(), localPath);
  } catch (IOException e) {
    LOG.error("Unable to record localization start for " + rsrc, e);
  }
  return rPath;
}
 

import org.apache.hadoop.yarn.api.records.LocalResource; //导入方法依赖的package包/类
/**
 * @return {@link Path} absolute path for localization which includes local
 *         directory path and the relative hierarchical path (if use local
 *         cache directory manager is enabled)
 * 
 * @param {@link LocalResourceRequest} Resource localization request to
 *        localize the resource.
 * @param {@link Path} local directory path
 * @param {@link DeletionService} Deletion Service to delete existing
 *        path for localization.
 */
@Override
public Path getPathForLocalization(LocalResourceRequest req,
    Path localDirPath, DeletionService delService) {
  Path rPath = localDirPath;
  if (useLocalCacheDirectoryManager && localDirPath != null) {

    if (!directoryManagers.containsKey(localDirPath)) {
      directoryManagers.putIfAbsent(localDirPath,
        new LocalCacheDirectoryManager(conf));
    }
    LocalCacheDirectoryManager dir = directoryManagers.get(localDirPath);

    rPath = localDirPath;
    String hierarchicalPath = dir.getRelativePathForLocalization();
    // For most of the scenarios we will get root path only which
    // is an empty string
    if (!hierarchicalPath.isEmpty()) {
      rPath = new Path(localDirPath, hierarchicalPath);
    }
    inProgressLocalResourcesMap.put(req, rPath);
  }

  while (true) {
    Path uniquePath = new Path(rPath,
        Long.toString(uniqueNumberGenerator.incrementAndGet()));
    File file = new File(uniquePath.toUri().getRawPath());
    if (!file.exists()) {
      rPath = uniquePath;
      break;
    }
    // If the directory already exists, delete it and move to next one.
    LOG.warn("Directory " + uniquePath + " already exists, " +
        "try next one.");
    if (delService != null) {
      delService.delete(getUser(), uniquePath);
    }
  }

  Path localPath = new Path(rPath, req.getPath().getName());
  LocalizedResource rsrc = localrsrc.get(req);
  rsrc.setLocalPath(localPath);
  LocalResource lr = LocalResource.newInstance(req.getResource(),
      req.getType(), req.getVisibility(), req.getSize(),
      req.getTimestamp());
  try {
    stateStore.startResourceLocalization(user, appId,
        ((LocalResourcePBImpl) lr).getProto(), localPath);
  } catch (IOException e) {
    LOG.error("Unable to record localization start for " + rsrc, e);
  }
  return rPath;
}