当前位置: 首页>>代码示例>>Java>>正文


Java FileContext.mkdir方法代码示例

本文整理汇总了Java中org.apache.hadoop.fs.FileContext.mkdir方法的典型用法代码示例。如果您正苦于以下问题:Java FileContext.mkdir方法的具体用法?Java FileContext.mkdir怎么用?Java FileContext.mkdir使用的例子?那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在org.apache.hadoop.fs.FileContext的用法示例。


在下文中一共展示了FileContext.mkdir方法的12个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: createTmpFile

import org.apache.hadoop.fs.FileContext; //导入方法依赖的package包/类
byte[] createTmpFile(Path dst, Random r, int len)
    throws IOException {
  // use unmodified local context
  FileContext lfs = FileContext.getLocalFSFileContext();
  dst = lfs.makeQualified(dst);
  lfs.mkdir(dst.getParent(), null, true);
  byte[] bytes = new byte[len];
  FSDataOutputStream out = null;
  try {
    out = lfs.create(dst, EnumSet.of(CREATE, OVERWRITE));
    r.nextBytes(bytes);
    out.write(bytes);
  } finally {
    if (out != null) out.close();
  }
  return bytes;
}
 
开发者ID:naver,项目名称:hadoop,代码行数:18,代码来源:TestDefaultContainerExecutor.java

示例2: createPath

import org.apache.hadoop.fs.FileContext; //导入方法依赖的package包/类
private Path createPath(FileContext fc, Path root, int year, int month,
                        int day, String id) throws IOException {
  Path path = new Path(root, year + Path.SEPARATOR + month + Path.SEPARATOR +
          day + Path.SEPARATOR + id);
  fc.mkdir(path, FsPermission.getDirDefault(), true);
  return path;
}
 
开发者ID:naver,项目名称:hadoop,代码行数:8,代码来源:TestJobHistoryUtils.java

示例3: mkdir

import org.apache.hadoop.fs.FileContext; //导入方法依赖的package包/类
private void mkdir(FileContext fc, Path path, FsPermission fsp)
    throws IOException {
  if (!fc.util().exists(path)) {
    try {
      fc.mkdir(path, fsp, true);

      FileStatus fsStatus = fc.getFileStatus(path);
      LOG.info("Perms after creating " + fsStatus.getPermission().toShort()
          + ", Expected: " + fsp.toShort());
      if (fsStatus.getPermission().toShort() != fsp.toShort()) {
        LOG.info("Explicitly setting permissions to : " + fsp.toShort()
            + ", " + fsp);
        fc.setPermission(path, fsp);
      }
    } catch (FileAlreadyExistsException e) {
      LOG.info("Directory: [" + path + "] already exists.");
    }
  }
}
 
开发者ID:naver,项目名称:hadoop,代码行数:20,代码来源:HistoryFileManager.java

示例4: setupForViewFsLocalFs

import org.apache.hadoop.fs.FileContext; //导入方法依赖的package包/类
static public FileContext setupForViewFsLocalFs(FileContextTestHelper helper) throws Exception {
  /**
   * create the test root on local_fs - the  mount table will point here
   */
  FileContext fsTarget = FileContext.getLocalFSFileContext();
  Path targetOfTests = helper.getTestRootPath(fsTarget);
  // In case previous test was killed before cleanup
  fsTarget.delete(targetOfTests, true);
  
  fsTarget.mkdir(targetOfTests, FileContext.DEFAULT_PERM, true);
  Configuration conf = new Configuration();
  
  // Set up viewfs link for test dir as described above
  String testDir = helper.getTestRootPath(fsTarget).toUri()
      .getPath();
  linkUpFirstComponents(conf, testDir, fsTarget, "test dir");
  
  
  // Set up viewfs link for home dir as described above
  setUpHomeDir(conf, fsTarget);
    
  // the test path may be relative to working dir - we need to make that work:
  // Set up viewfs link for wd as described above
  String wdDir = fsTarget.getWorkingDirectory().toUri().getPath();
  linkUpFirstComponents(conf, wdDir, fsTarget, "working dir");
  
  FileContext fc = FileContext.getFileContext(FsConstants.VIEWFS_URI, conf);
  fc.setWorkingDirectory(new Path(wdDir)); // in case testdir relative to wd.
  Log.info("Working dir is: " + fc.getWorkingDirectory());
  //System.out.println("SRCOfTests = "+ getTestRootPath(fc, "test"));
  //System.out.println("TargetOfTests = "+ targetOfTests.toUri());
  return fc;
}
 
开发者ID:nucypher,项目名称:hadoop-oss,代码行数:34,代码来源:ViewFsTestSetup.java

示例5: initializeLogDir

import org.apache.hadoop.fs.FileContext; //导入方法依赖的package包/类
private void initializeLogDir(FileContext lfs, String logDir) {
  try {
    lfs.mkdir(new Path(logDir), null, true);
  } catch (FileAlreadyExistsException fe) {
    // do nothing
  } catch (IOException e) {
    String msg = "Could not initialize log dir " + logDir;
    LOG.warn(msg, e);
    throw new YarnRuntimeException(msg, e);
  }
}
 
开发者ID:naver,项目名称:hadoop,代码行数:12,代码来源:ResourceLocalizationService.java

示例6: createDir

import org.apache.hadoop.fs.FileContext; //导入方法依赖的package包/类
private static void createDir(FileContext lfs, Path dirPath,
    FsPermission perms, boolean createParent) throws IOException {
  lfs.mkdir(dirPath, perms, createParent);
  if (!perms.equals(perms.applyUMask(lfs.getUMask()))) {
    lfs.setPermission(dirPath, perms);
  }
}
 
开发者ID:naver,项目名称:hadoop,代码行数:8,代码来源:ContainerLocalizer.java

示例7: createDir

import org.apache.hadoop.fs.FileContext; //导入方法依赖的package包/类
private void createDir(FileContext localFs, Path dir, FsPermission perm)
    throws IOException {
  if (dir == null) {
    return;
  }
  try {
    localFs.getFileStatus(dir);
  } catch (FileNotFoundException e) {
    createDir(localFs, dir.getParent(), perm);
    localFs.mkdir(dir, perm, false);
    if (!perm.equals(perm.applyUMask(localFs.getUMask()))) {
      localFs.setPermission(dir, perm);
    }
  }
}
 
开发者ID:naver,项目名称:hadoop,代码行数:16,代码来源:DirectoryCollection.java

示例8: clearDir

import org.apache.hadoop.fs.FileContext; //导入方法依赖的package包/类
private void clearDir(FileContext fc, Path p) throws IOException {
  try {
    fc.delete(p, true);
  } catch (FileNotFoundException e) {
      // ignore
  }
  fc.mkdir(p, FsPermission.getDirDefault(), false);
}
 
开发者ID:naver,项目名称:hadoop,代码行数:9,代码来源:TestJobHistoryUtils.java

示例9: createLogFile

import org.apache.hadoop.fs.FileContext; //导入方法依赖的package包/类
/**
 * Create simple log file
 * 
 * @return
 * @throws IOException
 */

private Path createLogFile() throws IOException {

  FileContext files = FileContext.getLocalFSFileContext();

  Path ws = new Path(workSpace.getAbsoluteFile().getAbsolutePath());

  files.delete(ws, true);
  Path workSpacePath = new Path(workSpace.getAbsolutePath(), "log");
  files.mkdir(workSpacePath, null, true);

  LOG.info("create logfile.log");
  Path logfile1 = new Path(workSpacePath, "logfile.log");

  FSDataOutputStream os = files.create(logfile1,
      EnumSet.of(CreateFlag.CREATE));
  os.writeBytes("4 3" + EL + "1 3" + EL + "4 44" + EL);
  os.writeBytes("2 3" + EL + "1 3" + EL + "0 45" + EL);
  os.writeBytes("4 3" + EL + "1 3" + EL + "1 44" + EL);

  os.flush();
  os.close();
  LOG.info("create logfile1.log");

  Path logfile2 = new Path(workSpacePath, "logfile1.log");

  os = files.create(logfile2, EnumSet.of(CreateFlag.CREATE));
  os.writeBytes("4 3" + EL + "1 3" + EL + "3 44" + EL);
  os.writeBytes("2 3" + EL + "1 3" + EL + "0 45" + EL);
  os.writeBytes("4 3" + EL + "1 3" + EL + "1 44" + EL);

  os.flush();
  os.close();

  return workSpacePath;
}
 
开发者ID:naver,项目名称:hadoop,代码行数:43,代码来源:TestLogalyzer.java

示例10: testGetPathForLocalization

import org.apache.hadoop.fs.FileContext; //导入方法依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void testGetPathForLocalization() throws Exception {
  FileContext lfs = FileContext.getLocalFSFileContext();
  Path base_path = new Path("target",
      TestLocalResourcesTrackerImpl.class.getSimpleName());
  final String user = "someuser";
  final ApplicationId appId = ApplicationId.newInstance(1, 1);
  Configuration conf = new YarnConfiguration();
  DrainDispatcher dispatcher = null;
  dispatcher = createDispatcher(conf);
  EventHandler<LocalizerEvent> localizerEventHandler =
      mock(EventHandler.class);
  EventHandler<LocalizerEvent> containerEventHandler =
      mock(EventHandler.class);
  dispatcher.register(LocalizerEventType.class, localizerEventHandler);
  dispatcher.register(ContainerEventType.class, containerEventHandler);
  NMStateStoreService stateStore = mock(NMStateStoreService.class);
  DeletionService delService = mock(DeletionService.class);
  try {
    LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1,
        LocalResourceVisibility.PUBLIC);
    LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
    ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc =
        new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
    localrsrc.put(req1, lr1);
    LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user,
        appId, dispatcher, localrsrc, true, conf, stateStore, null);
    Path conflictPath = new Path(base_path, "10");
    Path qualifiedConflictPath = lfs.makeQualified(conflictPath);
    lfs.mkdir(qualifiedConflictPath, null, true);
    Path rPath = tracker.getPathForLocalization(req1, base_path,
        delService);
    Assert.assertFalse(lfs.util().exists(rPath));
    verify(delService, times(1)).delete(eq(user), eq(conflictPath));
  } finally {
    lfs.delete(base_path, true);
    if (dispatcher != null) {
      dispatcher.stop();
    }
  }
}
 
开发者ID:naver,项目名称:hadoop,代码行数:43,代码来源:TestLocalResourcesTrackerImpl.java

示例11: testGetFullDirs

import org.apache.hadoop.fs.FileContext; //导入方法依赖的package包/类
@Test
public void testGetFullDirs() throws Exception {
  Configuration conf = new YarnConfiguration();

  conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
  FileContext localFs = FileContext.getLocalFSFileContext(conf);

  String localDir1 = new File(testDir, "localDir1").getPath();
  String localDir2 = new File(testDir, "localDir2").getPath();
  String logDir1 = new File(testDir, "logDir1").getPath();
  String logDir2 = new File(testDir, "logDir2").getPath();
  Path localDir1Path = new Path(localDir1);
  Path logDir1Path = new Path(logDir1);
  FsPermission dirPermissions = new FsPermission((short) 0410);
  localFs.mkdir(localDir1Path, dirPermissions, true);
  localFs.mkdir(logDir1Path, dirPermissions, true);

  conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir1 + "," + localDir2);
  conf.set(YarnConfiguration.NM_LOG_DIRS, logDir1 + "," + logDir2);
  conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE,
    0.0f);
  NodeManagerMetrics nm = NodeManagerMetrics.create();
  LocalDirsHandlerService dirSvc = new LocalDirsHandlerService(nm);
  dirSvc.init(conf);
  Assert.assertEquals(0, dirSvc.getLocalDirs().size());
  Assert.assertEquals(0, dirSvc.getLogDirs().size());
  Assert.assertEquals(1, dirSvc.getDiskFullLocalDirs().size());
  Assert.assertEquals(1, dirSvc.getDiskFullLogDirs().size());
  // check the metrics
  Assert.assertEquals(2, nm.getBadLocalDirs());
  Assert.assertEquals(2, nm.getBadLogDirs());
  Assert.assertEquals(0, nm.getGoodLocalDirsDiskUtilizationPerc());
  Assert.assertEquals(0, nm.getGoodLogDirsDiskUtilizationPerc());

  conf.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE,
    100.0f);
  nm = NodeManagerMetrics.create();
  dirSvc = new LocalDirsHandlerService(nm);
  dirSvc.init(conf);
  Assert.assertEquals(1, dirSvc.getLocalDirs().size());
  Assert.assertEquals(1, dirSvc.getLogDirs().size());
  Assert.assertEquals(0, dirSvc.getDiskFullLocalDirs().size());
  Assert.assertEquals(0, dirSvc.getDiskFullLogDirs().size());
  // check the metrics
  File dir = new File(localDir1);
  int utilizationPerc =
      (int) ((dir.getTotalSpace() - dir.getUsableSpace()) * 100 /
          dir.getTotalSpace());
  Assert.assertEquals(1, nm.getBadLocalDirs());
  Assert.assertEquals(1, nm.getBadLogDirs());
  Assert.assertEquals(utilizationPerc,
    nm.getGoodLocalDirsDiskUtilizationPerc());
  Assert
    .assertEquals(utilizationPerc, nm.getGoodLogDirsDiskUtilizationPerc());

  FileUtils.deleteDirectory(new File(localDir1));
  FileUtils.deleteDirectory(new File(localDir2));
  FileUtils.deleteDirectory(new File(logDir1));
  FileUtils.deleteDirectory(new File(logDir1));
  dirSvc.close();
}
 
开发者ID:naver,项目名称:hadoop,代码行数:62,代码来源:TestLocalDirsHandlerService.java

示例12: serviceInit

import org.apache.hadoop.fs.FileContext; //导入方法依赖的package包/类
@Override
public void serviceInit(Configuration conf) throws Exception {
  conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
  if (conf.get(MRJobConfig.MR_AM_STAGING_DIR) == null) {
    conf.set(MRJobConfig.MR_AM_STAGING_DIR, new File(getTestWorkDir(),
        "apps_staging_dir/").getAbsolutePath());
  }

  // By default, VMEM monitoring disabled, PMEM monitoring enabled.
  if (!conf.getBoolean(
      MRConfig.MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING,
      MRConfig.DEFAULT_MAPREDUCE_MINICLUSTER_CONTROL_RESOURCE_MONITORING)) {
    conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false);
    conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false);
  }

  conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,  "000");

  try {
    Path stagingPath = FileContext.getFileContext(conf).makeQualified(
        new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR)));
    /*
     * Re-configure the staging path on Windows if the file system is localFs.
     * We need to use a absolute path that contains the drive letter. The unit
     * test could run on a different drive than the AM. We can run into the
     * issue that job files are localized to the drive where the test runs on,
     * while the AM starts on a different drive and fails to find the job
     * metafiles. Using absolute path can avoid this ambiguity.
     */
    if (Path.WINDOWS) {
      if (LocalFileSystem.class.isInstance(stagingPath.getFileSystem(conf))) {
        conf.set(MRJobConfig.MR_AM_STAGING_DIR,
            new File(conf.get(MRJobConfig.MR_AM_STAGING_DIR))
                .getAbsolutePath());
      }
    }
    FileContext fc=FileContext.getFileContext(stagingPath.toUri(), conf);
    if (fc.util().exists(stagingPath)) {
      LOG.info(stagingPath + " exists! deleting...");
      fc.delete(stagingPath, true);
    }
    LOG.info("mkdir: " + stagingPath);
    //mkdir the staging directory so that right permissions are set while running as proxy user
    fc.mkdir(stagingPath, null, true);
    //mkdir done directory as well 
    String doneDir = JobHistoryUtils.getConfiguredHistoryServerDoneDirPrefix(conf);
    Path doneDirPath = fc.makeQualified(new Path(doneDir));
    fc.mkdir(doneDirPath, null, true);
  } catch (IOException e) {
    throw new YarnRuntimeException("Could not create staging directory. ", e);
  }
  conf.set(MRConfig.MASTER_ADDRESS, "test"); // The default is local because of
                                           // which shuffle doesn't happen
  //configure the shuffle service in NM
  conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
      new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID });
  conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT,
      ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class,
      Service.class);

  // Non-standard shuffle port
  conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);

  conf.setClass(YarnConfiguration.NM_CONTAINER_EXECUTOR,
      DefaultContainerExecutor.class, ContainerExecutor.class);

  // TestMRJobs is for testing non-uberized operation only; see TestUberAM
  // for corresponding uberized tests.
  conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);

  super.serviceInit(conf);
}
 
开发者ID:naver,项目名称:hadoop,代码行数:73,代码来源:MiniMRYarnCluster.java


注:本文中的org.apache.hadoop.fs.FileContext.mkdir方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。