Java LocalDirAllocator类代码示例

import org.apache.hadoop.fs.LocalDirAllocator; //导入依赖的package包/类
public MonitoringTimerTask(Configuration conf) throws YarnRuntimeException {
  float maxUsableSpacePercentagePerDisk =
      conf.getFloat(
        YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE,
        YarnConfiguration.DEFAULT_NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE);
  long minFreeSpacePerDiskMB =
      conf.getLong(YarnConfiguration.NM_MIN_PER_DISK_FREE_SPACE_MB,
        YarnConfiguration.DEFAULT_NM_MIN_PER_DISK_FREE_SPACE_MB);
  localDirs =
      new DirectoryCollection(
        validatePaths(conf
          .getTrimmedStrings(YarnConfiguration.NM_LOCAL_DIRS)),
        maxUsableSpacePercentagePerDisk, minFreeSpacePerDiskMB);
  logDirs =
      new DirectoryCollection(
        validatePaths(conf.getTrimmedStrings(YarnConfiguration.NM_LOG_DIRS)),
        maxUsableSpacePercentagePerDisk, minFreeSpacePerDiskMB);
  localDirsAllocator = new LocalDirAllocator(
      YarnConfiguration.NM_LOCAL_DIRS);
  logDirsAllocator = new LocalDirAllocator(YarnConfiguration.NM_LOG_DIRS);
}
 

import org.apache.hadoop.fs.LocalDirAllocator; //导入依赖的package包/类
public void setConf(Configuration conf) {
  if (conf instanceof JobConf) {
    this.conf = (JobConf) conf;
  } else {
    this.conf = new JobConf(conf);
  }
  this.mapOutputFile = ReflectionUtils.newInstance(
      conf.getClass(MRConfig.TASK_LOCAL_OUTPUT_CLASS,
        MROutputFiles.class, MapOutputFile.class), conf);
  this.lDirAlloc = new LocalDirAllocator(MRConfig.LOCAL_DIR);
  // add the static resolutions (this is required for the junit to
  // work on testcases that simulate multiple nodes on a single physical
  // node.
  String hostToResolved[] = conf.getStrings(MRConfig.STATIC_RESOLUTIONS);
  if (hostToResolved != null) {
    for (String str : hostToResolved) {
      String name = str.substring(0, str.indexOf('='));
      String resolvedName = str.substring(str.indexOf('=') + 1);
      NetUtils.addStaticResolution(name, resolvedName);
    }
  }
}
 

import org.apache.hadoop.fs.LocalDirAllocator; //导入依赖的package包/类
public OSSOutputStream(Configuration conf,
                       OSSFileSystem fs, String bucket, String key, Progressable progress,
                       FileSystem.Statistics statistics)
        throws IOException {
  this.bucket = bucket;
  this.key = key;
  this.progress = progress;
  this.fs = fs;
  this.statistics = statistics;

  lDirAlloc = new LocalDirAllocator(SmartOSSClientConfig.BUFFER_DIR);
  backupFile = lDirAlloc.createTmpFileForWrite("output-", LocalDirAllocator.SIZE_UNKNOWN, conf);
  closed = false;

  if (LOG.isDebugEnabled()) {
    LOG.debug("OutputStream for key '" + key + "' writing to tempfile: " +
            this.backupFile);
  }

  this.backupStream = new BufferedOutputStream(new FileOutputStream(backupFile));
}
 

import org.apache.hadoop.fs.LocalDirAllocator; //导入依赖的package包/类
/**
 * Default constructor
 *
 * @param account JOSS account object
 * @param url URL connection
 * @param contentTypeT content type
 * @param metadataT input metadata
 * @param connectionManager SwiftConnectionManager
 * @param fsT SwiftAPIClient
 * @throws IOException if error
 */
public SwiftNoStreamingOutputStream(JossAccount account, URL url, final String contentTypeT,
                         Map<String, String> metadataT, SwiftConnectionManager connectionManager,
                         SwiftAPIClient fsT)
        throws IOException {
  LOG.debug("SwiftNoStreamingOutputStream constructor entry for {}", url.toString());
  mUrl = url;
  contentType = contentTypeT;
  mAccount = account;
  scm = connectionManager;
  metadata = metadataT;
  fs = fsT;
  try {
    mBackupFile = fs.createTmpFileForWrite("output-",
        LocalDirAllocator.SIZE_UNKNOWN);

    LOG.debug("OutputStream for key '{}' writing to tempfile: {}", mUrl.toString(), mBackupFile);
    mBackupOutputStream = new BufferedOutputStream(new FileOutputStream(mBackupFile), 32768);
  } catch (IOException e) {
    LOG.error(e.getMessage());
    throw e;
  }

}
 

import org.apache.hadoop.fs.LocalDirAllocator; //导入依赖的package包/类
public JobLocalizer(JobConf ttConf, String user, String jobid,
    String... localDirs) throws IOException {
  if (null == user) {
    throw new IOException("Cannot initialize for null user");
  }
  this.user = user;
  if (null == jobid) {
    throw new IOException("Cannot initialize for null jobid");
  }
  this.jobid = jobid;
  this.ttConf = new JobConf(ttConf);
  lfs = FileSystem.getLocal(this.ttConf).getRaw();
  this.localDirs = createPaths(user, localDirs);
  this.ttConf.setStrings(JOB_LOCAL_CTXT, localDirs);
  Collections.shuffle(this.localDirs);
  lDirAlloc = new LocalDirAllocator(JOB_LOCAL_CTXT);
  JOBDIR = TaskTracker.JOBCACHE + Path.SEPARATOR + jobid;
  DISTDIR = JOBDIR + "/" + TaskTracker.DISTCACHEDIR;
  WORKDIR = JOBDIR + "/work";
  JARDST = JOBDIR + "/" + TaskTracker.JARSDIR + "/job.jar";
  JOBCONF = JOBDIR + "/" + TaskTracker.JOBFILE;
  JOBTOKEN = JOBDIR + "/" + TaskTracker.JOB_TOKEN_FILE;
}
 

import org.apache.hadoop.fs.LocalDirAllocator; //导入依赖的package包/类
public void setConf(Configuration conf) {
  if (conf instanceof JobConf) {
    this.conf = (JobConf) conf;
  } else {
    this.conf = new JobConf(conf);
  }
  this.mapOutputFile.setConf(this.conf);
  this.lDirAlloc = new LocalDirAllocator("mapred.local.dir");
  // add the static resolutions (this is required for the junit to
  // work on testcases that simulate multiple nodes on a single physical
  // node.
  String hostToResolved[] = conf.getStrings("hadoop.net.static.resolutions");
  if (hostToResolved != null) {
    for (String str : hostToResolved) {
      String name = str.substring(0, str.indexOf('='));
      String resolvedName = str.substring(str.indexOf('=') + 1);
      NetUtils.addStaticResolution(name, resolvedName);
    }
  }
}
 

import org.apache.hadoop.fs.LocalDirAllocator; //导入依赖的package包/类
/**
 * Write the child's configuration to the disk and set it in configuration so
 * that the child can pick it up from there.
 * 
 * @param lDirAlloc
 * @throws IOException
 */
void setupChildTaskConfiguration(LocalDirAllocator lDirAlloc)
    throws IOException {

  Path localTaskFile =
      lDirAlloc.getLocalPathForWrite(TaskTracker.getTaskConfFile(
          t.getUser(), t.getJobID().toString(), t.getTaskID().toString(), t
              .isTaskCleanupTask()), conf);

  // write the child's task configuration file to the local disk
  JobLocalizer.writeLocalJobFile(localTaskFile, conf);

  // Set the final job file in the task. The child needs to know the correct
  // path to job.xml. So set this path accordingly.
  t.setJobFile(localTaskFile.toString());
}
 

import org.apache.hadoop.fs.LocalDirAllocator; //导入依赖的package包/类
static void localizeTask(Task task, JobConf jobConf, String logLocation) 
throws IOException{
  
  // Do the task-type specific localization
  task.localizeConfiguration(jobConf);
  
  // Write files required to re-run the task with IsolationRunner
  if (task.supportIsolationRunner(jobConf)) {
    task.writeFilesRequiredForRerun(jobConf);
  }
  
  //write the localized task jobconf
  LocalDirAllocator lDirAlloc = 
    new LocalDirAllocator(JobConf.MAPRED_LOCAL_DIR_PROPERTY);
  Path localTaskFile =
    lDirAlloc.getLocalPathForWrite(TaskTracker.JOBFILE, jobConf);
  JobLocalizer.writeLocalJobFile(localTaskFile, jobConf);
  task.setJobFile(localTaskFile.toString());
  task.setConf(jobConf);
}
 

import org.apache.hadoop.fs.LocalDirAllocator; //导入依赖的package包/类
@Override
public void writeFilesRequiredForRerun(JobConf conf)
    throws IOException {
  // split.info file is used only by IsolationRunner.
  // Write the split file to the local disk if it is a normal map task (not a
  // job-setup or a job-cleanup task) and if the user wishes to run
  // IsolationRunner either by setting keep.failed.tasks.files to true or by
  // using keep.tasks.files.pattern
  if (isMapOrReduce()) {
    // localize the split meta-information
    Path localSplitMeta =
      new LocalDirAllocator("mapred.local.dir").getLocalPathForWrite(
          TaskTracker.getLocalSplitFile(conf.getUser(), getJobID()
              .toString(), getTaskID().toString()), conf);
    LOG.debug("Writing local split to " + localSplitMeta);
    DataOutputStream out = FileSystem.getLocal(conf).create(localSplitMeta);
    splitMetaInfo.write(out);
    out.close();
  }
}
 

import org.apache.hadoop.fs.LocalDirAllocator; //导入依赖的package包/类
@Override
public void setup(LocalDirAllocator allocator, LocalStorage localStorage)
    throws IOException {

  // Check the permissions of the task-controller binary by running
  // it plainly.  If permissions are correct, it returns an error
  // code 1, else it returns 24 or something else if some other bugs
  // are also present.
  String[] taskControllerCmd =
      new String[] { taskControllerExe };
  ShellCommandExecutor shExec = new ShellCommandExecutor(taskControllerCmd);
  try {
    shExec.execute();
  } catch (ExitCodeException e) {
    int exitCode = shExec.getExitCode();
    if (exitCode != 1) {
      LOG.warn("Exit code from checking binary permissions is : " + exitCode);
      logOutput(shExec.getOutput());
      throw new IOException("Task controller setup failed because of invalid"
        + "permissions/ownership with exit code " + exitCode, e);
    }
  }
  this.allocator = allocator;
  this.localStorage = localStorage;
}
 

import org.apache.hadoop.fs.LocalDirAllocator; //导入依赖的package包/类
public TrackerDistributedCacheManager(Configuration conf,
    TaskController controller) throws IOException {
  this.localFs = FileSystem.getLocal(conf);
  this.trackerConf = conf;
  this.lDirAllocator = new LocalDirAllocator("mapred.local.dir");

  // setting the cache size to a default of 10GB
  this.allowedCacheSize = conf.getLong
    ("local.cache.size", DEFAULT_CACHE_SIZE);
  // setting the cache number of subdirectories limit to a default of 10000
  this.allowedCacheSubdirs = conf.getLong
    ("mapreduce.tasktracker.cache.local.numberdirectories",
     DEFAULT_CACHE_SUBDIR_LIMIT);
  double cleanupPct = conf.getFloat("mapreduce.tasktracker.cache.local.keep.pct",
      DEFAULT_CACHE_KEEP_AROUND_PCT);
  this.allowedCacheSizeCleanupGoal = 
    (long)(this.allowedCacheSize * cleanupPct);
  this.allowedCacheSubdirsCleanupGoal = 
    (long)(this.allowedCacheSubdirs * cleanupPct);

  this.taskController = controller;
  this.cleanupThread = new CleanupThread(conf);
}
 

import org.apache.hadoop.fs.LocalDirAllocator; //导入依赖的package包/类
private TaskTracker setupTaskTracker(Configuration conf) throws Exception {
  JobConf ttConf = new JobConf(conf);
  // Doesn't matter what we give here - we won't actually
  // connect to it.
  TaskTracker tt = new TaskTracker();
  tt.setConf(ttConf);
  tt.setTaskController(Mockito.mock(TaskController.class));
  LocalDirAllocator localDirAllocator = 
    new LocalDirAllocator("mapred.local.dir");
  tt.setLocalDirAllocator(localDirAllocator);
  LocalFileSystem localFs = FileSystem.getLocal(conf);
  LocalStorage localStorage = new LocalStorage(ttConf.getLocalDirs());
  localStorage.checkDirs(localFs, true);
  tt.setLocalStorage(localStorage);
  tt.setLocalFileSystem(localFs);
  tt.initializeDirectories();
  return tt;
}
 

import org.apache.hadoop.fs.LocalDirAllocator; //导入依赖的package包/类
public TestJvmManager() throws Exception {
  user = UserGroupInformation.getCurrentUser().getShortUserName();
  tt = new TaskTracker();
  ttConf = new JobConf();
  ttConf.setLong("mapred.tasktracker.tasks.sleeptime-before-sigkill", 2000);
  tt.setConf(ttConf);
  tt.setMaxMapSlots(MAP_SLOTS);
  tt.setMaxReduceSlots(REDUCE_SLOTS);
  TaskController dtc;
  tt.setTaskController((dtc = new DefaultTaskController()));
  Configuration conf = new Configuration();
  dtc.setConf(conf);
  LocalDirAllocator ldirAlloc =
      new LocalDirAllocator(JobConf.MAPRED_LOCAL_DIR_PROPERTY);
  tt.getTaskController().setup(ldirAlloc, new LocalStorage(ttConf.getLocalDirs()));
  JobID jobId = new JobID("test", 0);
  jvmManager = new JvmManager(tt);
  tt.setJvmManagerInstance(jvmManager);
  tt.setUserLogManager(new UserLogManager(ttConf));
  tt.setCleanupThread(new InlineCleanupQueue());
}
 

import org.apache.hadoop.fs.LocalDirAllocator; //导入依赖的package包/类
private void startTT(JobConf conf) throws IOException, InterruptedException {
  myClock = new FakeClock(); // clock is reset.
  String localdirs = TEST_ROOT_DIR + "/userlogs/local/0," + 
  					TEST_ROOT_DIR + "/userlogs/local/1";
  conf.set(JobConf.MAPRED_LOCAL_DIR_PROPERTY, localdirs);
  tt = new TaskTracker();
  tt.setConf(new JobConf(conf));
  LocalDirAllocator localDirAllocator = 
    new LocalDirAllocator("mapred.local.dir");
  tt.setLocalDirAllocator(localDirAllocator);
  LocalStorage localStorage = new LocalStorage(conf.getLocalDirs());
  LocalFileSystem localFs = FileSystem.getLocal(conf);
  localStorage.checkDirs(localFs, true);
  tt.setLocalStorage(localStorage);
  localizer = new Localizer(FileSystem.get(conf), conf
      .getTrimmedStrings(JobConf.MAPRED_LOCAL_DIR_PROPERTY));
  tt.setLocalizer(localizer);
  userLogManager = new UtilsForTests.InLineUserLogManager(conf);
  TaskController taskController = userLogManager.getTaskController();
  taskController.setup(localDirAllocator, localStorage);
  tt.setTaskController(taskController);
  userLogCleaner = userLogManager.getUserLogCleaner();
  userLogCleaner.setClock(myClock);
  tt.setUserLogManager(userLogManager);
  userLogManager.clearOldUserLogs(conf);
}
 

import org.apache.hadoop.fs.LocalDirAllocator; //导入依赖的package包/类
@SuppressWarnings("deprecation")
public CoronaJobTrackerRunner(
    TaskTracker.TaskInProgress tip, Task task, TaskTracker tracker,
    JobConf ttConf, CoronaSessionInfo info, String originalPath,
    String releasePath) throws IOException {
  super(tip, task, tracker, ttConf);
  this.coronaSessionInfo = info;
  this.originalPath = originalPath;
  this.releasePath = releasePath;
  LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir");

  workDir = new File(lDirAlloc.getLocalPathForWrite(
      TaskTracker.getLocalTaskDir(
          task.getJobID().toString(),
          task.getTaskID().toString(),
          task.isTaskCleanupTask())
          + Path.SEPARATOR + MRConstants.WORKDIR,
          conf). toString());
  if (!workDir.mkdirs()) {
    if (!workDir.isDirectory()) {
      throw new IOException("Mkdirs failed to create " + workDir.toString());
    }
  }
  localizeTaskConfiguration(tracker, ttConf, workDir.toString(), task, task
      .getJobID());
}