Java DeletionService类代码示例

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入依赖的package包/类
public ResourceLocalizationService(Dispatcher dispatcher,
    ContainerExecutor exec, DeletionService delService,
    LocalDirsHandlerService dirsHandler, Context context) {

  super(ResourceLocalizationService.class.getName());
  this.exec = exec;
  this.dispatcher = dispatcher;
  this.delService = delService;
  this.dirsHandler = dirsHandler;

  this.cacheCleanup = new ScheduledThreadPoolExecutor(1,
      new ThreadFactoryBuilder()
        .setNameFormat("ResourceLocalizationService Cache Cleanup")
        .build());
  this.stateStore = context.getNMStateStore();
  this.nmContext = context;
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入依赖的package包/类
private void cleanUpLocalDir(FileContext lfs, DeletionService del,
    String localDir) {
  long currentTimeStamp = System.currentTimeMillis();
  renameLocalDir(lfs, localDir, ContainerLocalizer.USERCACHE,
    currentTimeStamp);
  renameLocalDir(lfs, localDir, ContainerLocalizer.FILECACHE,
    currentTimeStamp);
  renameLocalDir(lfs, localDir, ResourceLocalizationService.NM_PRIVATE_DIR,
    currentTimeStamp);
  try {
    deleteLocalDir(lfs, del, localDir);
  } catch (IOException e) {
    // Do nothing, just give the warning
    LOG.warn("Failed to delete localDir: " + localDir);
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入依赖的package包/类
private void deleteLocalDir(FileContext lfs, DeletionService del,
    String localDir) throws IOException {
  RemoteIterator<FileStatus> fileStatus = lfs.listStatus(new Path(localDir));
  if (fileStatus != null) {
    while (fileStatus.hasNext()) {
      FileStatus status = fileStatus.next();
      try {
        if (status.getPath().getName().matches(".*" +
            ContainerLocalizer.USERCACHE + "_DEL_.*")) {
          LOG.info("usercache path : " + status.getPath().toString());
          cleanUpFilesPerUserDir(lfs, del, status.getPath());
        } else if (status.getPath().getName()
            .matches(".*" + NM_PRIVATE_DIR + "_DEL_.*")
            ||
            status.getPath().getName()
                .matches(".*" + ContainerLocalizer.FILECACHE + "_DEL_.*")) {
          del.delete(null, status.getPath(), new Path[] {});
        }
      } catch (IOException ex) {
        // Do nothing, just give the warning
        LOG.warn("Failed to delete this local Directory: " +
            status.getPath().getName());
      }
    }
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入依赖的package包/类
private void cleanUpFilesPerUserDir(FileContext lfs, DeletionService del,
    Path userDirPath) throws IOException {
  RemoteIterator<FileStatus> userDirStatus = lfs.listStatus(userDirPath);
  FileDeletionTask dependentDeletionTask =
      del.createFileDeletionTask(null, userDirPath, new Path[] {});
  if (userDirStatus != null && userDirStatus.hasNext()) {
    List<FileDeletionTask> deletionTasks = new ArrayList<FileDeletionTask>();
    while (userDirStatus.hasNext()) {
      FileStatus status = userDirStatus.next();
      String owner = status.getOwner();
      FileDeletionTask deletionTask =
          del.createFileDeletionTask(owner, null,
            new Path[] { status.getPath() });
      deletionTask.addFileDeletionTaskDependency(dependentDeletionTask);
      deletionTasks.add(deletionTask);
    }
    for (FileDeletionTask task : deletionTasks) {
      del.scheduleFileDeletionTask(task);
    }
  } else {
    del.scheduleFileDeletionTask(dependentDeletionTask);
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入依赖的package包/类
@Override
 public boolean remove(LocalizedResource rem, DeletionService delService) {
// current synchronization guaranteed by crude RLS event for cleanup
   LocalizedResource rsrc = localrsrc.get(rem.getRequest());
   if (null == rsrc) {
     LOG.error("Attempt to remove absent resource: " + rem.getRequest()
         + " from " + getUser());
     return true;
   }
   if (rsrc.getRefCount() > 0
       || ResourceState.DOWNLOADING.equals(rsrc.getState()) || rsrc != rem) {
     // internal error
     LOG.error("Attempt to remove resource: " + rsrc
         + " with non-zero refcount");
     return false;
   } else { // ResourceState is LOCALIZED or INIT
     if (ResourceState.LOCALIZED.equals(rsrc.getState())) {
       delService.delete(getUser(), getPathToDelete(rsrc.getLocalPath()));
     }
     removeResource(rem.getRequest());
     LOG.info("Removed " + rsrc.getLocalPath() + " from localized cache");
     return true;
   }
 }
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入依赖的package包/类
private ResourceLocalizationService createSpyService(
    DrainDispatcher dispatcher, LocalDirsHandlerService dirsHandler,
    NMStateStoreService stateStore) {
  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalizerTracker mockLocalizerTracker = mock(LocalizerTracker.class);
  DeletionService delService = mock(DeletionService.class);
  NMContext nmContext =
      new NMContext(new NMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInNM(), null,
        new ApplicationACLsManager(conf), stateStore);
  ResourceLocalizationService rawService =
    new ResourceLocalizationService(dispatcher, exec, delService,
                                    dirsHandler, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  doReturn(mockLocalizerTracker).when(spyService).createLocalizerTracker(
      isA(Configuration.class));
  doReturn(lfs).when(spyService)
      .getLocalFileContext(isA(Configuration.class));
  return spyService;
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入依赖的package包/类
@Test
public void testHandlingApplicationFinishedEvent() throws IOException {
  DeletionService delService = new DeletionService(null);
  NonAggregatingLogHandler aggregatingLogHandler =
      new NonAggregatingLogHandler(new InlineDispatcher(),
          delService,
          dirsHandler,
          new NMNullStateStoreService());

  dirsHandler.init(conf);
  dirsHandler.start();
  delService.init(conf);
  delService.start();
  aggregatingLogHandler.init(conf);
  aggregatingLogHandler.start();
  
  // It should NOT throw RejectedExecutionException
  aggregatingLogHandler.handle(new LogHandlerAppFinishedEvent(appId));
  aggregatingLogHandler.stop();

  // It should NOT throw RejectedExecutionException after stopping
  // handler service.
  aggregatingLogHandler.handle(new LogHandlerAppFinishedEvent(appId));
  aggregatingLogHandler.close();
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入依赖的package包/类
/**
 * Function to verify that the DeletionService object received the right
 * requests.
 * 
 * @param delService the DeletionService mock which we verify against
 * 
 * @param user the user name to use when verifying the deletion
 * 
 * @param timeout amount in milliseconds to wait before we decide the calls
 * didn't come through
 * 
 * @param matchPaths the paths to match in the delete calls
 * 
 * @throws WantedButNotInvoked if the calls could not be verified
 */
static void testDeletionServiceCall(DeletionService delService, String user,
    long timeout, Path... matchPaths) {

  long verifyStartTime = System.currentTimeMillis();
  WantedButNotInvoked notInvokedException = null;
  boolean matched = false;
  while (!matched && System.currentTimeMillis() < verifyStartTime + timeout) {
    try {
      verify(delService).delete(eq(user), (Path) eq(null),
        Mockito.argThat(new DeletePathsMatcher(matchPaths)));
      matched = true;
    } catch (WantedButNotInvoked e) {
      notInvokedException = e;
      try {
        Thread.sleep(50l);
      } catch (InterruptedException i) {
      }
    }
  }
  if (!matched) {
    throw notInvokedException;
  }
  return;
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入依赖的package包/类
@Test(timeout=20000)
public void testStopAfterError() throws Exception {
  DeletionService delSrvc = mock(DeletionService.class);

  // get the AppLogAggregationImpl thread to crash
  LocalDirsHandlerService mockedDirSvc = mock(LocalDirsHandlerService.class);
  when(mockedDirSvc.getLogDirs()).thenThrow(new RuntimeException());
  
  LogAggregationService logAggregationService =
      new LogAggregationService(dispatcher, this.context, delSrvc,
                                mockedDirSvc);
  logAggregationService.init(this.conf);
  logAggregationService.start();

  ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);
  logAggregationService.handle(new LogHandlerAppStartedEvent(
          application1, this.user, null,
          ContainerLogsRetentionPolicy.ALL_CONTAINERS, this.acls));

  logAggregationService.stop();
  assertEquals(0, logAggregationService.getNumAggregators());
  logAggregationService.close();
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入依赖的package包/类
private ContainerManagerImpl createContainerManager(Context context,
    DeletionService delSrvc) {
  return new ContainerManagerImpl(context, exec, delSrvc,
      mock(NodeStatusUpdater.class), metrics, dirsHandler) {
    @Override
    public void
    setBlockNewContainerRequests(boolean blockNewContainerRequests) {
      // do nothing
    }
    @Override
    protected void authorizeGetAndStopContainerRequest(
        ContainerId containerId, Container container,
        boolean stopRequest, NMTokenIdentifier identifier)
        throws YarnException {
      if(container == null || container.getUser().equals("Fail")){
        throw new YarnException("Reject this container");
      }
    }
  };
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入依赖的package包/类
@Test(timeout=20000)
public void testStopAfterError() throws Exception {
  DeletionService delSrvc = mock(DeletionService.class);

  // get the AppLogAggregationImpl thread to crash
  LocalDirsHandlerService mockedDirSvc = mock(LocalDirsHandlerService.class);
  when(mockedDirSvc.getLogDirs()).thenThrow(new RuntimeException());
  
  LogAggregationService logAggregationService =
      new LogAggregationService(dispatcher, this.context, delSrvc,
                                mockedDirSvc);
  logAggregationService.init(this.conf);
  logAggregationService.start();

  ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);
  LogAggregationContext contextWithAllContainers =
      Records.newRecord(LogAggregationContext.class);
  contextWithAllContainers.setLogAggregationPolicyClassName(
      AllContainerLogAggregationPolicy.class.getName());
  logAggregationService.handle(new LogHandlerAppStartedEvent(
      application1, this.user, null, this.acls, contextWithAllContainers));

  logAggregationService.stop();
  assertEquals(0, logAggregationService.getNumAggregators());
  logAggregationService.close();
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入依赖的package包/类
private ResourceLocalizationService createSpyService(
    DrainDispatcher dispatcher, LocalDirsHandlerService dirsHandler,
    NMStateStoreService stateStore) {
  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalizerTracker mockLocalizerTracker = mock(LocalizerTracker.class);
  DeletionService delService = mock(DeletionService.class);
  NMContext nmContext =
      new NMContext(new NMContainerTokenSecretManager(conf),
        new NMTokenSecretManagerInNM(), null,
        new ApplicationACLsManager(conf), stateStore,null);
  ResourceLocalizationService rawService =
    new ResourceLocalizationService(dispatcher, exec, delService,
                                    dirsHandler, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  doReturn(mockLocalizerTracker).when(spyService).createLocalizerTracker(
      isA(Configuration.class));
  doReturn(lfs).when(spyService)
      .getLocalFileContext(isA(Configuration.class));
  return spyService;
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入依赖的package包/类
@Test
public void testHandlingApplicationFinishedEvent() throws IOException {
  DeletionService delService = new DeletionService(null);
  NonAggregatingLogHandler aggregatingLogHandler =
      new NonAggregatingLogHandler(new InlineDispatcher(),
          delService,
          dirsHandler);

  dirsHandler.init(conf);
  dirsHandler.start();
  delService.init(conf);
  delService.start();
  aggregatingLogHandler.init(conf);
  aggregatingLogHandler.start();
  
  // It should NOT throw RejectedExecutionException
  aggregatingLogHandler.handle(new LogHandlerAppFinishedEvent(appId));
  aggregatingLogHandler.stop();

  // It should NOT throw RejectedExecutionException after stopping
  // handler service.
  aggregatingLogHandler.handle(new LogHandlerAppFinishedEvent(appId));
  aggregatingLogHandler.close();
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入依赖的package包/类
private void cleanUpLocalDir(FileContext lfs, DeletionService del) {
  long currentTimeStamp = System.currentTimeMillis();
  for (String localDir : dirsHandler.getLocalDirs()) {
    renameLocalDir(lfs, localDir, ContainerLocalizer.USERCACHE,
        currentTimeStamp);
    renameLocalDir(lfs, localDir, ContainerLocalizer.FILECACHE,
        currentTimeStamp);
    renameLocalDir(lfs, localDir, ResourceLocalizationService.NM_PRIVATE_DIR,
        currentTimeStamp);
    try {
      deleteLocalDir(lfs, del, localDir);
    } catch (IOException e) {
      // Do nothing, just give the warning
      LOG.warn("Failed to delete localDir: " + localDir);
    }
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入依赖的package包/类
private void cleanUpFilesPerUserDir(FileContext lfs, DeletionService del,
    Path userDirPath) throws IOException {
  RemoteIterator<FileStatus> userDirStatus = lfs.listStatus(userDirPath);
  FileDeletionTask dependentDeletionTask =
      del.createFileDeletionTask(null, userDirPath, new Path[] {});
  if (userDirStatus != null) {
    List<FileDeletionTask> deletionTasks = new ArrayList<FileDeletionTask>();
    while (userDirStatus.hasNext()) {
      FileStatus status = userDirStatus.next();
      String owner = status.getOwner();
      FileDeletionTask deletionTask =
          del.createFileDeletionTask(owner, null,
            new Path[] { status.getPath() });
      deletionTask.addFileDeletionTaskDependency(dependentDeletionTask);
      deletionTasks.add(deletionTask);
    }
    for (FileDeletionTask task : deletionTasks) {
      del.scheduleFileDeletionTask(task);
    }
  } else {
    del.scheduleFileDeletionTask(dependentDeletionTask);
  }
}