Java LocalResourceVisibility.PUBLIC屬性代碼示例

private Path copy(Path sCopy, Path dstdir) throws IOException {
  FileSystem sourceFs = sCopy.getFileSystem(conf);
  Path dCopy = new Path(dstdir, "tmp_"+sCopy.getName());
  FileStatus sStat = sourceFs.getFileStatus(sCopy);
  if (sStat.getModificationTime() != resource.getTimestamp()) {
    throw new IOException("Resource " + sCopy +
        " changed on src filesystem (expected " + resource.getTimestamp() +
        ", was " + sStat.getModificationTime());
  }
  if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) {
    if (!isPublic(sourceFs, sCopy, sStat, statCache)) {
      throw new IOException("Resource " + sCopy +
          " is not publicly accessable and as such cannot be part of the" +
          " public cache.");
    }
  }

  FileUtil.copy(sourceFs, sStat, FileSystem.getLocal(conf), dCopy, false,
      true, conf);
  return dCopy;
}
 

LocalResourcesTracker createMockTracker(String user, final long rsrcSize,
    long nRsrcs, long timestamp, long tsstep) {
  Configuration conf = new Configuration();
  ConcurrentMap<LocalResourceRequest,LocalizedResource> trackerResources =
    new ConcurrentHashMap<LocalResourceRequest,LocalizedResource>();
  LocalResourcesTracker ret = spy(new LocalResourcesTrackerImpl(user, null,
    null, trackerResources, false, conf, new NMNullStateStoreService(),null));
  for (int i = 0; i < nRsrcs; ++i) {
    final LocalResourceRequest req = new LocalResourceRequest(
        new Path("file:///" + user + "/rsrc" + i), timestamp + i * tsstep,
        LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, null);
    final long ts = timestamp + i * tsstep;
    final Path p = new Path("file:///local/" + user + "/rsrc" + i);
    LocalizedResource rsrc = new LocalizedResource(req, null) {
      @Override public int getRefCount() { return 0; }
      @Override public long getSize() { return rsrcSize; }
      @Override public Path getLocalPath() { return p; }
      @Override public long getTimestamp() { return ts; }
      @Override
      public ResourceState getState() { return ResourceState.LOCALIZED; }
    };
    trackerResources.put(req, rsrc);
  }
  return ret;
}
 

private void changePermissions(FileSystem fs, final Path path)
    throws IOException, InterruptedException {
  File f = new File(path.toUri());
  if (FileUtils.isSymlink(f)) {
    // avoid following symlinks when changing permissions
    return;
  }
  boolean isDir = f.isDirectory();
  FsPermission perm = cachePerms;
  // set public perms as 755 or 555 based on dir or file
  if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) {
    perm = isDir ? PUBLIC_DIR_PERMS : PUBLIC_FILE_PERMS;
  }
  // set private perms as 700 or 500
  else {
    // PRIVATE:
    // APPLICATION:
    perm = isDir ? PRIVATE_DIR_PERMS : PRIVATE_FILE_PERMS;
  }
  LOG.debug("Changing permissions for path " + path + " to perm " + perm);
  final FsPermission fPerm = perm;
  if (null == userUgi) {
    files.setPermission(path, perm);
  } else {
    userUgi.doAs(new PrivilegedExceptionAction<Void>() {
      public Void run() throws Exception {
        files.setPermission(path, fPerm);
        return null;
      }
    });
  }
  if (isDir) {
    FileStatus[] statuses = fs.listStatus(path);
    for (FileStatus status : statuses) {
      changePermissions(fs, status.getPath());
    }
  }
}
 

LocalResourcesTracker createMockTracker(String user, final long rsrcSize,
    long nRsrcs, long timestamp, long tsstep) {
  Configuration conf = new Configuration();
  ConcurrentMap<LocalResourceRequest,LocalizedResource> trackerResources =
    new ConcurrentHashMap<LocalResourceRequest,LocalizedResource>();
  LocalResourcesTracker ret = spy(new LocalResourcesTrackerImpl(user, null,
    null, trackerResources, false, conf, new NMNullStateStoreService()));
  for (int i = 0; i < nRsrcs; ++i) {
    final LocalResourceRequest req = new LocalResourceRequest(
        new Path("file:///" + user + "/rsrc" + i), timestamp + i * tsstep,
        LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, null);
    final long ts = timestamp + i * tsstep;
    final Path p = new Path("file:///local/" + user + "/rsrc" + i);
    LocalizedResource rsrc = new LocalizedResource(req, null) {
      @Override public int getRefCount() { return 0; }
      @Override public long getSize() { return rsrcSize; }
      @Override public Path getLocalPath() { return p; }
      @Override public long getTimestamp() { return ts; }
      @Override
      public ResourceState getState() { return ResourceState.LOCALIZED; }
    };
    trackerResources.put(req, rsrc);
  }
  return ret;
}
 

public LocalResourceVisibility getVisibility(Configuration conf, URI uri,
    Map<URI, FileStatus> statCache) throws IOException {
  if (isPublic(conf, uri, statCache)) {
    return LocalResourceVisibility.PUBLIC;
  } else {
    return LocalResourceVisibility.PRIVATE;
  }
}
 

/**
 * Recursively change permissions of all files/dirs on path based 
 * on resource visibility.
 * Change to 755 or 700 for dirs, 555 or 500 for files.
 * @param fs FileSystem
 * @param path Path to modify perms for
 * @throws IOException
 * @throws InterruptedException 
 */
private void changePermissions(FileSystem fs, final Path path)
    throws IOException, InterruptedException {
  File f = new File(path.toUri());
  if (FileUtils.isSymlink(f)) {
    // avoid following symlinks when changing permissions
    return;
  }
  boolean isDir = f.isDirectory();
  FsPermission perm = cachePerms;
  // set public perms as 755 or 555 based on dir or file
  if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) {
    perm = isDir ? PUBLIC_DIR_PERMS : PUBLIC_FILE_PERMS;
  }
  // set private perms as 700 or 500
  else {
    // PRIVATE:
    // APPLICATION:
    perm = isDir ? PRIVATE_DIR_PERMS : PRIVATE_FILE_PERMS;
  }
  LOG.debug("Changing permissions for path " + path
      + " to perm " + perm);
  final FsPermission fPerm = perm;
  if (null == userUgi) {
    files.setPermission(path, perm);
  }
  else {
    userUgi.doAs(new PrivilegedExceptionAction<Void>() {
      public Void run() throws Exception {
        files.setPermission(path, fPerm);
        return null;
      }
    });
  }
  if (isDir) {
    FileStatus[] statuses = fs.listStatus(path);
    for (FileStatus status : statuses) {
      changePermissions(fs, status.getPath());
    }
  }
}
 

private void verifyPermsRecursively(FileSystem fs,
    FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  FileStatus status = files.getFileStatus(p);
  if (status.isDirectory()) {
    if (vis == LocalResourceVisibility.PUBLIC) {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PUBLIC_DIR_PERMS.toShort());
    }
    else {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PRIVATE_DIR_PERMS.toShort());
    }
    if (!status.isSymlink()) {
      FileStatus[] statuses = fs.listStatus(p);
      for (FileStatus stat : statuses) {
        verifyPermsRecursively(fs, files, stat.getPath(), vis);
      }
    }
  }
  else {
    if (vis == LocalResourceVisibility.PUBLIC) {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PUBLIC_FILE_PERMS.toShort());
    }
    else {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PRIVATE_FILE_PERMS.toShort());
    }
  }      
}
 

/**
 * Checks that the (original) remote file is either owned by the user who
 * started the app or public.
 */
@VisibleForTesting
boolean verifyAccess() throws IOException {
  // if it is in the public cache, it's trivially OK
  if (resource.getVisibility() == LocalResourceVisibility.PUBLIC) {
    return true;
  }

  final Path remotePath;
  try {
    remotePath = ConverterUtils.getPathFromYarnURL(resource.getResource());
  } catch (URISyntaxException e) {
    throw new IOException("Invalid resource", e);
  }

  // get the file status of the HDFS file
  FileSystem remoteFs = remotePath.getFileSystem(conf);
  FileStatus status = remoteFs.getFileStatus(remotePath);
  // check to see if the file has been modified in any way
  if (status.getModificationTime() != resource.getTimestamp()) {
    LOG.warn("The remote file " + remotePath +
        " has changed since it's localized; will not consider it for upload");
    return false;
  }

  // check for the user ownership
  if (status.getOwner().equals(user)) {
    return true; // the user owns the file
  }
  // check if the file is publicly readable otherwise
  return fileIsPublic(remotePath, remoteFs, status);
}
 

@Test(timeout=10000)
@SuppressWarnings("unchecked")
public void testConsistency() {
  String user = "testuser";
  DrainDispatcher dispatcher = null;
  try {
    Configuration conf = new Configuration();
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);

    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
    LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.PUBLIC);
    LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req1, lr1);
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        null, dispatcher, localrsrc, false, conf,
        new NMNullStateStoreService(), null);

    ResourceEvent req11Event = new ResourceRequestEvent(req1,
        LocalResourceVisibility.PUBLIC, lc1);

    ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);

    // Localize R1 for C1
    tracker.handle(req11Event);

    dispatcher.await();

    // Verify refCount for R1 is 1
    Assert.assertEquals(1, lr1.getRefCount());

    dispatcher.await();
    verifyTrackedResourceCount(tracker, 1);

    // Localize resource1
    ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path(
        "file:///tmp/r1"), 1);
    lr1.handle(rle);
    Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
    Assert.assertTrue(createdummylocalizefile(new Path("file:///tmp/r1")));
    LocalizedResource rsrcbefore = tracker.iterator().next();
    File resFile = new File(lr1.getLocalPath().toUri().getRawPath()
        .toString());
    Assert.assertTrue(resFile.exists());
    Assert.assertTrue(resFile.delete());

    // Localize R1 for C1
    tracker.handle(req11Event);

    dispatcher.await();
    lr1.handle(rle);
    Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
    LocalizedResource rsrcafter = tracker.iterator().next();
    if (rsrcbefore == rsrcafter) {
      Assert.fail("Localized resource should not be equal");
    }
    // Release resource1
    tracker.handle(rel11Event);
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 

@SuppressWarnings("unchecked")
@Test
public void testResourcePresentInGoodDir() throws IOException {
  String user = "testuser";
  DrainDispatcher dispatcher = null;
  try {
    Configuration conf = new Configuration();
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler =
        mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler =
        mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);

    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
    LocalResourceRequest req1 =
        createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
    LocalResourceRequest req2 =
        createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC);
    LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
    LocalizedResource lr2 = createLocalizedResource(req2, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc =
        new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req1, lr1);
    localrsrc.put(req2, lr2);
    LocalDirsHandlerService dirsHandler = mock(LocalDirsHandlerService.class);
    List<String> goodDirs = new ArrayList<String>();
    // /tmp/somedir2 is bad
    goodDirs.add("/tmp/somedir1/");
    goodDirs.add("/tmp/somedir2");
    Mockito.when(dirsHandler.getLocalDirs()).thenReturn(goodDirs);
    Mockito.when(dirsHandler.getLocalDirsForRead()).thenReturn(goodDirs);
    LocalResourcesTrackerImpl tracker =
        new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc,
            true , conf, new NMNullStateStoreService(), dirsHandler);
    ResourceEvent req11Event =
        new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
    ResourceEvent req21Event =
        new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1);
    // Localize R1 for C1
    tracker.handle(req11Event);
    // Localize R2 for C1
    tracker.handle(req21Event);
    dispatcher.await();
    // Localize resource1
    Path p1 = tracker.getPathForLocalization(req1,
        new Path("/tmp/somedir1"), null);
    Path p2 = tracker.getPathForLocalization(req2,
        new Path("/tmp/somedir2"), null);
    ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(req1, p1, 1);
    tracker.handle(rle1);
    ResourceLocalizedEvent rle2 = new ResourceLocalizedEvent(req2, p2, 1);
    tracker.handle(rle2);
    dispatcher.await();
    // Remove somedir2 from gooddirs
    Assert.assertTrue(tracker.checkLocalResource(lr2));
    goodDirs.remove(1);
    Assert.assertFalse(tracker.checkLocalResource(lr2));
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 

@Test (timeout=10000)
public void testDownloadBadPublic() throws IOException, URISyntaxException,
    InterruptedException {
  Configuration conf = new Configuration();
  conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
  FileContext files = FileContext.getLocalFSFileContext(conf);
  final Path basedir = files.makeQualified(new Path("target",
    TestFSDownload.class.getSimpleName()));
  files.mkdir(basedir, null, true);
  conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
  
  Map<LocalResource, LocalResourceVisibility> rsrcVis =
      new HashMap<LocalResource, LocalResourceVisibility>();

  Random rand = new Random();
  long sharedSeed = rand.nextLong();
  rand.setSeed(sharedSeed);
  System.out.println("SEED: " + sharedSeed);

  Map<LocalResource,Future<Path>> pending =
    new HashMap<LocalResource,Future<Path>>();
  ExecutorService exec = Executors.newSingleThreadExecutor();
  LocalDirAllocator dirs =
    new LocalDirAllocator(TestFSDownload.class.getName());
  int size = 512;
  LocalResourceVisibility vis = LocalResourceVisibility.PUBLIC;
  Path path = new Path(basedir, "test-file");
  LocalResource rsrc = createFile(files, path, size, rand, vis);
  rsrcVis.put(rsrc, vis);
  Path destPath = dirs.getLocalPathForWrite(
      basedir.toString(), size, conf);
  destPath = new Path (destPath,
    Long.toString(uniqueNumberGenerator.incrementAndGet()));
  FSDownload fsd =
    new FSDownload(files, UserGroupInformation.getCurrentUser(), conf,
        destPath, rsrc);
  pending.put(rsrc, exec.submit(fsd));
  exec.shutdown();
  while (!exec.awaitTermination(1000, TimeUnit.MILLISECONDS));
  Assert.assertTrue(pending.get(rsrc).isDone());

  try {
    for (Map.Entry<LocalResource,Future<Path>> p : pending.entrySet()) {
      p.getValue().get();
      Assert.fail("We localized a file that is not public.");
    }
  } catch (ExecutionException e) {
    Assert.assertTrue(e.getCause() instanceof IOException);
  }
}
 

@Test
@SuppressWarnings("unchecked")
public void testReleaseWhileDownloading() throws Exception {
  String user = "testuser";
  DrainDispatcher dispatcher = null;
  try {
    Configuration conf = new Configuration();
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler =
        mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler =
        mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);

    ContainerId cId = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalizerContext lc = new LocalizerContext(user, cId, null);

    LocalResourceRequest req =
        createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
    LocalizedResource lr = createLocalizedResource(req, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc =
        new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req, lr);
    LocalResourcesTracker tracker =
        new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc,
            false, conf, new NMNullStateStoreService(), null);

    // request the resource
    ResourceEvent reqEvent =
        new ResourceRequestEvent(req, LocalResourceVisibility.PUBLIC, lc);
    tracker.handle(reqEvent);

    // release the resource
    ResourceEvent relEvent = new ResourceReleaseEvent(req, cId);
    tracker.handle(relEvent);

    // download completing after release
    ResourceLocalizedEvent rle =
        new ResourceLocalizedEvent(req, new Path("file:///tmp/r1"), 1);
    tracker.handle(rle);

    dispatcher.await();
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 

@Test(timeout=10000)
@SuppressWarnings("unchecked")
public void testConsistency() {
  String user = "testuser";
  DrainDispatcher dispatcher = null;
  try {
    Configuration conf = new Configuration();
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);

    ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
    LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
    LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.PUBLIC);
    LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req1, lr1);
    LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
        null, dispatcher, localrsrc, false, conf,
        new NMNullStateStoreService());

    ResourceEvent req11Event = new ResourceRequestEvent(req1,
        LocalResourceVisibility.PUBLIC, lc1);

    ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);

    // Localize R1 for C1
    tracker.handle(req11Event);

    dispatcher.await();

    // Verify refCount for R1 is 1
    Assert.assertEquals(1, lr1.getRefCount());

    dispatcher.await();
    verifyTrackedResourceCount(tracker, 1);

    // Localize resource1
    ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path(
        "file:///tmp/r1"), 1);
    lr1.handle(rle);
    Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
    Assert.assertTrue(createdummylocalizefile(new Path("file:///tmp/r1")));
    LocalizedResource rsrcbefore = tracker.iterator().next();
    File resFile = new File(lr1.getLocalPath().toUri().getRawPath()
        .toString());
    Assert.assertTrue(resFile.exists());
    Assert.assertTrue(resFile.delete());

    // Localize R1 for C1
    tracker.handle(req11Event);

    dispatcher.await();
    lr1.handle(rle);
    Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
    LocalizedResource rsrcafter = tracker.iterator().next();
    if (rsrcbefore == rsrcafter) {
      Assert.fail("Localized resource should not be equal");
    }
    // Release resource1
    tracker.handle(rel11Event);
  } finally {
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}