Java FileContext类代码示例

import org.apache.hadoop.fs.FileContext; //导入依赖的package包/类
@Before
public void setUp() throws Exception {
  // create the test root on local_fs
  fcTarget = FileContext.getLocalFSFileContext();
  chrootedTo = fileContextTestHelper.getAbsoluteTestRootPath(fcTarget);
  // In case previous test was killed before cleanup
  fcTarget.delete(chrootedTo, true);
  
  fcTarget.mkdir(chrootedTo, FileContext.DEFAULT_PERM, true);

  Configuration conf = new Configuration();

  // ChRoot to the root of the testDirectory
  fc = FileContext.getFileContext(
      new ChRootedFs(fcTarget.getDefaultFileSystem(), chrootedTo), conf);
}
 

import org.apache.hadoop.fs.FileContext; //导入依赖的package包/类
@Override
public void close() throws IOException {
  // Output the result to a file Results in the output dir
  FileContext fc;
  try {
    fc = FileContext.getFileContext(jobConf);
  } catch (IOException ioe) {
    System.err.println("Can not initialize the file system: " + 
        ioe.getLocalizedMessage());
    return;
  }
  FSDataOutputStream o = fc.create(FileOutputFormat.getTaskOutputPath(jobConf, "Results"),
      EnumSet.of(CreateFlag.CREATE));
     
  PrintStream out = new PrintStream(o);
  printResults(out);
  out.close();
  o.close();
}
 

import org.apache.hadoop.fs.FileContext; //导入依赖的package包/类
@Test
public void testResolvePathThroughMountPoints() throws IOException {
  fileContextTestHelper.createFile(fcView, "/user/foo");
  Assert.assertEquals(new Path(targetTestRoot,"user/foo"),
                        fcView.resolvePath(new Path("/user/foo")));
  
  fcView.mkdir(
      fileContextTestHelper.getTestRootPath(fcView, "/user/dirX"),
      FileContext.DEFAULT_PERM, false);
  Assert.assertEquals(new Path(targetTestRoot,"user/dirX"),
      fcView.resolvePath(new Path("/user/dirX")));

  
  fcView.mkdir(
      fileContextTestHelper.getTestRootPath(fcView, "/user/dirX/dirY"),
      FileContext.DEFAULT_PERM, false);
  Assert.assertEquals(new Path(targetTestRoot,"user/dirX/dirY"),
      fcView.resolvePath(new Path("/user/dirX/dirY")));
}
 

import org.apache.hadoop.fs.FileContext; //导入依赖的package包/类
/** Main function called by tool runner.
 * It first initializes data by parsing the command line arguments.
 * It then calls the loadGenerator
 */
@Override
public int run(String[] args) throws Exception {
  int exitCode = parseArgsMR(args);
  if (exitCode != 0) {
    return exitCode;
  }
  System.out.println("Running LoadGeneratorMR against fileSystem: " + 
  FileContext.getFileContext().getDefaultFileSystem().getUri());

  return submitAsMapReduce(); // reducer will print the results
}
 

import org.apache.hadoop.fs.FileContext; //导入依赖的package包/类
@Test
public void testMkdirDelete() throws IOException {
  fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "/dirX"), FileContext.DEFAULT_PERM, false);
  Assert.assertTrue(isDir(fc, new Path("/dirX")));
  Assert.assertTrue(isDir(fcTarget, new Path(chrootedTo,"dirX")));
  
  fc.mkdir(fileContextTestHelper.getTestRootPath(fc, "/dirX/dirY"), FileContext.DEFAULT_PERM, false);
  Assert.assertTrue(isDir(fc, new Path("/dirX/dirY")));
  Assert.assertTrue(isDir(fcTarget, new Path(chrootedTo,"dirX/dirY")));
  

  // Delete the created dir
  Assert.assertTrue(fc.delete(new Path("/dirX/dirY"), false));
  Assert.assertFalse(exists(fc, new Path("/dirX/dirY")));
  Assert.assertFalse(exists(fcTarget, new Path(chrootedTo,"dirX/dirY")));
  
  Assert.assertTrue(fc.delete(new Path("/dirX"), false));
  Assert.assertFalse(exists(fc, new Path("/dirX")));
  Assert.assertFalse(exists(fcTarget, new Path(chrootedTo,"dirX")));
  
}
 

import org.apache.hadoop.fs.FileContext; //导入依赖的package包/类
@Test
public void testRename() throws IOException {
  // Rename a file
  fileContextTestHelper.createFile(fc, "/newDir/foo");
  fc.rename(new Path("/newDir/foo"), new Path("/newDir/fooBar"));
  Assert.assertFalse(exists(fc, new Path("/newDir/foo")));
  Assert.assertFalse(exists(fcTarget, new Path(chrootedTo,"newDir/foo")));
  Assert.assertTrue(isFile(fc, fileContextTestHelper.getTestRootPath(fc,"/newDir/fooBar")));
  Assert.assertTrue(isFile(fcTarget, new Path(chrootedTo,"newDir/fooBar")));
  
  
  // Rename a dir
  fc.mkdir(new Path("/newDir/dirFoo"), FileContext.DEFAULT_PERM, false);
  fc.rename(new Path("/newDir/dirFoo"), new Path("/newDir/dirFooBar"));
  Assert.assertFalse(exists(fc, new Path("/newDir/dirFoo")));
  Assert.assertFalse(exists(fcTarget, new Path(chrootedTo,"newDir/dirFoo")));
  Assert.assertTrue(isDir(fc, fileContextTestHelper.getTestRootPath(fc,"/newDir/dirFooBar")));
  Assert.assertTrue(isDir(fcTarget, new Path(chrootedTo,"newDir/dirFooBar")));
}
 

import org.apache.hadoop.fs.FileContext; //导入依赖的package包/类
@Before
public void initJunitModeTest() throws Exception {
  LOG.info("initJunitModeTest");

  conf = new HdfsConfiguration();
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, blockSize); // 100K
                                                            // blocksize

  cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
  cluster.waitActive();

  mfs = cluster.getFileSystem();
  mfc = FileContext.getFileContext();

  Path rootdir = new Path(ROOT_DIR);
  mfs.mkdirs(rootdir);
}
 

import org.apache.hadoop.fs.FileContext; //导入依赖的package包/类
private void doRenameLinkTargetNotWritableFC() throws Exception {
  // Rename the link when the target and parent are not writable
  user.doAs(new PrivilegedExceptionAction<Object>() {
    @Override
    public Object run() throws IOException {
      // First FileContext
      FileContext myfc = FileContext.getFileContext(conf);
      Path newlink = new Path(linkParent, "newlink");
      myfc.rename(link, newlink, Rename.NONE);
      Path linkTarget = myfc.getLinkTarget(newlink);
      assertEquals("Expected link's target to match target!",
          target, linkTarget);
      return null;
    }
  });
  assertTrue("Expected target to exist", wrapper.exists(target));
}
 

import org.apache.hadoop.fs.FileContext; //导入依赖的package包/类
private void mkdir(FileContext fc, Path path, FsPermission fsp)
    throws IOException {
  if (!fc.util().exists(path)) {
    try {
      fc.mkdir(path, fsp, true);

      FileStatus fsStatus = fc.getFileStatus(path);
      LOG.info("Perms after creating " + fsStatus.getPermission().toShort()
          + ", Expected: " + fsp.toShort());
      if (fsStatus.getPermission().toShort() != fsp.toShort()) {
        LOG.info("Explicitly setting permissions to : " + fsp.toShort()
            + ", " + fsp);
        fc.setPermission(path, fsp);
      }
    } catch (FileAlreadyExistsException e) {
      LOG.info("Directory: [" + path + "] already exists.");
    }
  }
}
 

import org.apache.hadoop.fs.FileContext; //导入依赖的package包/类
@Test
public void testNonSecureRunAsSubmitter() throws Exception {
  Assume.assumeTrue(shouldRun());
  Assume.assumeFalse(UserGroupInformation.isSecurityEnabled());
  String expectedRunAsUser = appSubmitter;
  conf.set(YarnConfiguration.NM_NONSECURE_MODE_LIMIT_USERS, "false");
  exec.setConf(conf);
  File touchFile = new File(workSpace, "touch-file");
  int ret = runAndBlock("touch", touchFile.getAbsolutePath());

  assertEquals(0, ret);
  FileStatus fileStatus =
      FileContext.getLocalFSFileContext().getFileStatus(
        new Path(touchFile.getAbsolutePath()));
  assertEquals(expectedRunAsUser, fileStatus.getOwner());
  cleanupAppFiles(expectedRunAsUser);
  // reset conf
  conf.unset(YarnConfiguration.NM_NONSECURE_MODE_LIMIT_USERS);
  exec.setConf(conf);
}
 

import org.apache.hadoop.fs.FileContext; //导入依赖的package包/类
static LocalResource createJar(FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  LOG.info("Create jar file " + p);
  File jarFile = new File((files.makeQualified(p)).toUri());
  FileOutputStream stream = new FileOutputStream(jarFile);
  LOG.info("Create jar out stream ");
  JarOutputStream out = new JarOutputStream(stream, new Manifest());
  LOG.info("Done writing jar stream ");
  out.close();
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(p));
  FileStatus status = files.getFileStatus(p);
  ret.setSize(status.getLen());
  ret.setTimestamp(status.getModificationTime());
  ret.setType(LocalResourceType.PATTERN);
  ret.setVisibility(vis);
  ret.setPattern("classes/.*");
  return ret;
}
 

import org.apache.hadoop.fs.FileContext; //导入依赖的package包/类
static LocalResource createJarFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException,
    URISyntaxException {
  byte[] bytes = new byte[len];
  r.nextBytes(bytes);

  File archiveFile = new File(p.toUri().getPath() + ".jar");
  archiveFile.createNewFile();
  JarOutputStream out = new JarOutputStream(
      new FileOutputStream(archiveFile));
  out.putNextEntry(new JarEntry(p.getName()));
  out.write(bytes);
  out.closeEntry();
  out.close();

  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
      + ".jar")));
  ret.setSize(len);
  ret.setType(LocalResourceType.ARCHIVE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".jar"))
      .getModificationTime());
  return ret;
}
 

import org.apache.hadoop.fs.FileContext; //导入依赖的package包/类
@Before
public void setUp() throws Exception {
  fcTarget = fc;
  fcTarget2 = fc2;
  targetTestRoot = fileContextTestHelper.getAbsoluteTestRootPath(fc);
  targetTestRoot2 = fileContextTestHelper.getAbsoluteTestRootPath(fc2);

  fcTarget.delete(targetTestRoot, true);
  fcTarget2.delete(targetTestRoot2, true);
  fcTarget.mkdir(targetTestRoot, new FsPermission((short) 0750), true);
  fcTarget2.mkdir(targetTestRoot2, new FsPermission((short) 0750), true);

  fsViewConf = ViewFileSystemTestSetup.createConfig();
  setupMountPoints();
  fcView = FileContext.getFileContext(FsConstants.VIEWFS_URI, fsViewConf);
}
 

import org.apache.hadoop.fs.FileContext; //导入依赖的package包/类
private void doRenameSrcNotWritableFC() throws Exception {
  // Rename the link when the target and parent are not writable
  try {
    user.doAs(new PrivilegedExceptionAction<Object>() {
      @Override
      public Object run() throws IOException {
        FileContext myfc = FileContext.getFileContext(conf);
        Path newlink = new Path(targetParent, "newlink");
        myfc.rename(link, newlink, Rename.NONE);
        return null;
      }
    });
    fail("Renamed link even though link's parent is not writable!");
  } catch (IOException e) {
    GenericTestUtils.assertExceptionContains("Permission denied", e);
  }
}
 

import org.apache.hadoop.fs.FileContext; //导入依赖的package包/类
private void cleanUpLocalDir(FileContext lfs, DeletionService del,
    String localDir) {
  long currentTimeStamp = System.currentTimeMillis();
  renameLocalDir(lfs, localDir, ContainerLocalizer.USERCACHE,
    currentTimeStamp);
  renameLocalDir(lfs, localDir, ContainerLocalizer.FILECACHE,
    currentTimeStamp);
  renameLocalDir(lfs, localDir, ResourceLocalizationService.NM_PRIVATE_DIR,
    currentTimeStamp);
  try {
    deleteLocalDir(lfs, del, localDir);
  } catch (IOException e) {
    // Do nothing, just give the warning
    LOG.warn("Failed to delete localDir: " + localDir);
  }
}