Java MRAsyncDiskService类代码示例

import org.apache.hadoop.mapreduce.util.MRAsyncDiskService; //导入依赖的package包/类
@Test
/** Test that volumes specified as relative paths are handled properly
 * by MRAsyncDiskService (MAPREDUCE-1887).
 */
public void testVolumeNormalization() throws Throwable {
  LOG.info("TEST_ROOT_DIR is " + TEST_ROOT_DIR);

  String relativeTestRoot = relativeToWorking(TEST_ROOT_DIR);

  FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
  String [] vols = new String[] { relativeTestRoot + "/0",
      relativeTestRoot + "/1" };

  // Put a file in one of the volumes to be cleared on startup.
  Path delDir = new Path(vols[0], MRAsyncDiskService.TOBEDELETED);
  localFileSystem.mkdirs(delDir);
  localFileSystem.create(new Path(delDir, "foo")).close();

  MRAsyncDiskService service = new MRAsyncDiskService(
      localFileSystem, vols);
  makeSureCleanedUp(vols, service);
}
 

import org.apache.hadoop.mapreduce.util.MRAsyncDiskService; //导入依赖的package包/类
/**
 * This test creates some directories inside the toBeDeleted directory and
 * then start the asyncDiskService.
 * AsyncDiskService will create tasks to delete the content inside the
 * toBeDeleted directories.
 */
@Test
public void testMRAsyncDiskServiceStartupCleaning() throws Throwable {
  FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
  String[] vols = new String[]{TEST_ROOT_DIR + "/0",
      TEST_ROOT_DIR + "/1"};

  String a = "a";
  String b = "b";
  String c = "b/c";
  String d = "d";
  
  // Create directories inside SUBDIR
  String suffix = Path.SEPARATOR_CHAR + MRAsyncDiskService.TOBEDELETED;
  File fa = new File(vols[0] + suffix, a);
  File fb = new File(vols[1] + suffix, b);
  File fc = new File(vols[1] + suffix, c);
  File fd = new File(vols[1] + suffix, d);
  
  // Create the directories
  fa.mkdirs();
  fb.mkdirs();
  fc.mkdirs();
  fd.mkdirs();

  assertTrue(fa.exists());
  assertTrue(fb.exists());
  assertTrue(fc.exists());
  assertTrue(fd.exists());
  
  // Create the asyncDiskService which will delete all contents inside SUBDIR
  MRAsyncDiskService service = new MRAsyncDiskService(
      localFileSystem, vols);
  
  // Make sure everything is cleaned up
  makeSureCleanedUp(vols, service);
}
 

import org.apache.hadoop.mapreduce.util.MRAsyncDiskService; //导入依赖的package包/类
private void makeSureCleanedUp(String[] vols, MRAsyncDiskService service)
    throws Throwable {
  // Sleep at most 5 seconds to make sure the deleted items are all gone.
  service.shutdown();
  if (!service.awaitTermination(5000)) {
    fail("MRAsyncDiskService is still not shutdown in 5 seconds!");
  }
  
  // All contents should be gone by now.
  for (int i = 0; i < vols.length; i++) {
    File subDir = new File(vols[0]);
    String[] subDirContent = subDir.list();
    assertEquals("Volume should contain a single child: "
        + MRAsyncDiskService.TOBEDELETED, 1, subDirContent.length);
    
    File toBeDeletedDir = new File(vols[0], MRAsyncDiskService.TOBEDELETED);
    String[] content = toBeDeletedDir.list();
    assertNotNull("Cannot find " + toBeDeletedDir, content);
    assertEquals("" + toBeDeletedDir + " should be empty now.", 0,
        content.length);
  }
}
 

import org.apache.hadoop.mapreduce.util.MRAsyncDiskService; //导入依赖的package包/类
/**
 * Delete a local path with asyncDiskService if available,
 * or otherwise synchronously with local file system.
 */
private static void deleteLocalPath(MRAsyncDiskService asyncDiskService,
    LocalFileSystem fs, Path path) throws IOException {
  boolean deleted = false;
  if (asyncDiskService != null) {
    // Try to delete using asyncDiskService
    String localPathToDelete = 
      path.toUri().getPath();
    deleted = asyncDiskService.moveAndDeleteAbsolutePath(localPathToDelete);
    if (!deleted) {
      LOG.warn("Cannot find DistributedCache path " + localPathToDelete
          + " on any of the asyncDiskService volumes!");
    }
  }
  if (!deleted) {
    // If no asyncDiskService, we will delete the files synchronously
    fs.delete(path, true);
  }
  LOG.info("Deleted path " + path);
}
 

import org.apache.hadoop.mapreduce.util.MRAsyncDiskService; //导入依赖的package包/类
private void initializeTracker() throws IOException {
  tracker.setIndexCache(new IndexCache(trackerFConf));
  tracker.setTaskMemoryManagerEnabledFlag();

  // for test case system FS is the local FS
  tracker.systemFS = FileSystem.getLocal(trackerFConf);
  tracker.setLocalFileSystem(tracker.systemFS);
  tracker.systemDirectory = new Path(TEST_ROOT_DIR.getAbsolutePath());

  tracker.runningTasks = new LinkedHashMap<TaskAttemptID, TaskInProgress>();
  tracker.runningJobs = new TreeMap<JobID, RunningJob>();
  tracker.setAsyncDiskService(new MRAsyncDiskService(trackerFConf));
  tracker.getAsyncDiskService().cleanupAllVolumes();

  // Set up TaskTracker instrumentation
  tracker.setTaskTrackerInstrumentation(
      TaskTracker.createInstrumentation(tracker, trackerFConf));

  // setup task controller
  taskController = createTaskController();
  taskController.setConf(trackerFConf);
  taskController.setup();
  tracker.setTaskController(taskController);
  tracker.setLocalizer(new Localizer(tracker.getLocalFileSystem(), localDirs,
      taskController));
}
 

import org.apache.hadoop.mapreduce.util.MRAsyncDiskService; //导入依赖的package包/类
@Test
public void testToleratesSomeUnwritableVolumes() throws Throwable {
  FileSystem localFileSystem = FileSystem.getLocal(new Configuration());
  String[] vols = new String[]{TEST_ROOT_DIR + "/0",
      TEST_ROOT_DIR + "/1"};
  
  assertTrue(new File(vols[0]).mkdirs());
  assertEquals(0, FileUtil.chmod(vols[0], "400")); // read only
  try {
    new MRAsyncDiskService(localFileSystem, vols);
  } finally {
    FileUtil.chmod(vols[0], "755"); // make writable again
  }
}