Java DeletionService.start方法代码示例

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入方法依赖的package包/类
@Test
public void testHandlingApplicationFinishedEvent() throws IOException {
  DeletionService delService = new DeletionService(null);
  NonAggregatingLogHandler aggregatingLogHandler =
      new NonAggregatingLogHandler(new InlineDispatcher(),
          delService,
          dirsHandler,
          new NMNullStateStoreService());

  dirsHandler.init(conf);
  dirsHandler.start();
  delService.init(conf);
  delService.start();
  aggregatingLogHandler.init(conf);
  aggregatingLogHandler.start();
  
  // It should NOT throw RejectedExecutionException
  aggregatingLogHandler.handle(new LogHandlerAppFinishedEvent(appId));
  aggregatingLogHandler.stop();

  // It should NOT throw RejectedExecutionException after stopping
  // handler service.
  aggregatingLogHandler.handle(new LogHandlerAppFinishedEvent(appId));
  aggregatingLogHandler.close();
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入方法依赖的package包/类
@Test
public void testHandlingApplicationFinishedEvent() throws IOException {
  DeletionService delService = new DeletionService(null);
  NonAggregatingLogHandler aggregatingLogHandler =
      new NonAggregatingLogHandler(new InlineDispatcher(),
          delService,
          dirsHandler);

  dirsHandler.init(conf);
  dirsHandler.start();
  delService.init(conf);
  delService.start();
  aggregatingLogHandler.init(conf);
  aggregatingLogHandler.start();
  
  // It should NOT throw RejectedExecutionException
  aggregatingLogHandler.handle(new LogHandlerAppFinishedEvent(appId));
  aggregatingLogHandler.stop();

  // It should NOT throw RejectedExecutionException after stopping
  // handler service.
  aggregatingLogHandler.handle(new LogHandlerAppFinishedEvent(appId));
  aggregatingLogHandler.close();
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入方法依赖的package包/类
@Test( timeout = 10000)
@SuppressWarnings("unchecked") // mocked generics
public void testLocalizerRunnerException() throws Exception {
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
  dispatcher.register(ApplicationEventType.class, applicationBus);
  EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
  dispatcher.register(ContainerEventType.class, containerBus);

  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
  LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler);
  dirsHandlerSpy.init(conf);

  DeletionService delServiceReal = new DeletionService(exec);
  DeletionService delService = spy(delServiceReal);
  delService.init(new Configuration());
  delService.start();

  ResourceLocalizationService rawService =
      new ResourceLocalizationService(dispatcher, exec, delService,
      dirsHandlerSpy, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  try {
    spyService.init(conf);
    spyService.start();

    // init application
    final Application app = mock(Application.class);
    final ApplicationId appId =
        BuilderUtils.newApplicationId(314159265358979L, 3);
    when(app.getUser()).thenReturn("user0");
    when(app.getAppId()).thenReturn(appId);
    spyService.handle(new ApplicationLocalizationEvent(
        LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
    dispatcher.await();

    Random r = new Random();
    long seed = r.nextLong();
    System.out.println("SEED: " + seed);
    r.setSeed(seed);
    final Container c = getMockContainer(appId, 42, "user0");
    final LocalResource resource1 = getPrivateMockedResource(r);
    System.out.println("Here 4");
    
    final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
    Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
      new HashMap<LocalResourceVisibility, 
                  Collection<LocalResourceRequest>>();
    List<LocalResourceRequest> privateResourceList =
        new ArrayList<LocalResourceRequest>();
    privateResourceList.add(req1);
    rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);

    final Constructor<?>[] constructors =
        FSError.class.getDeclaredConstructors();
    constructors[0].setAccessible(true);
    FSError fsError =
        (FSError) constructors[0].newInstance(new IOException("Disk Error"));

    Mockito
      .doThrow(fsError)
      .when(dirsHandlerSpy)
      .getLocalPathForWrite(isA(String.class));
    spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
    Thread.sleep(1000);
    dispatcher.await();
    // Verify if ContainerResourceFailedEvent is invoked on FSError
    verify(containerBus).handle(isA(ContainerResourceFailedEvent.class));
  } finally {
    spyService.stop();
    dispatcher.stop();
    delService.stop();
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入方法依赖的package包/类
@Test
public void testLocalizationInit() throws Exception {
  conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(new Configuration());

  ContainerExecutor exec = mock(ContainerExecutor.class);
  DeletionService delService = spy(new DeletionService(exec));
  delService.init(conf);
  delService.start();

  List<Path> localDirs = new ArrayList<Path>();
  String[] sDirs = new String[4];
  for (int i = 0; i < 4; ++i) {
    localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
    sDirs[i] = localDirs.get(i).toString();
  }
  conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);

  LocalDirsHandlerService diskhandler = new LocalDirsHandlerService();
  diskhandler.init(conf);

  ResourceLocalizationService locService =
    spy(new ResourceLocalizationService(dispatcher, exec, delService,
                                        diskhandler, nmContext));
  doReturn(lfs)
    .when(locService).getLocalFileContext(isA(Configuration.class));
  try {
    dispatcher.start();

    // initialize ResourceLocalizationService
    locService.init(conf);

    final FsPermission defaultPerm = new FsPermission((short)0755);

    // verify directory creation
    for (Path p : localDirs) {
      p = new Path((new URI(p.toString())).getPath());
      Path usercache = new Path(p, ContainerLocalizer.USERCACHE);
      verify(spylfs)
        .mkdir(eq(usercache),
            eq(defaultPerm), eq(true));
      Path publicCache = new Path(p, ContainerLocalizer.FILECACHE);
      verify(spylfs)
        .mkdir(eq(publicCache),
            eq(defaultPerm), eq(true));
      Path nmPriv = new Path(p, ResourceLocalizationService.NM_PRIVATE_DIR);
      verify(spylfs).mkdir(eq(nmPriv),
          eq(ResourceLocalizationService.NM_PRIVATE_PERM), eq(true));
    }
  } finally {
    dispatcher.stop();
    delService.stop();
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入方法依赖的package包/类
@Test
public void testDirectoryCleanupOnNewlyCreatedStateStore()
    throws IOException, URISyntaxException {
  conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(new Configuration());

  ContainerExecutor exec = mock(ContainerExecutor.class);
  DeletionService delService = spy(new DeletionService(exec));
  delService.init(conf);
  delService.start();

  List<Path> localDirs = new ArrayList<Path>();
  String[] sDirs = new String[4];
  for (int i = 0; i < 4; ++i) {
    localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
    sDirs[i] = localDirs.get(i).toString();
  }
  conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);

  LocalDirsHandlerService diskhandler = new LocalDirsHandlerService();
  diskhandler.init(conf);

  NMStateStoreService nmStateStoreService = mock(NMStateStoreService.class);
  when(nmStateStoreService.canRecover()).thenReturn(true);
  when(nmStateStoreService.isNewlyCreated()).thenReturn(true);

  ResourceLocalizationService locService =
      spy(new ResourceLocalizationService(dispatcher, exec, delService,
          diskhandler,nmContext));
  doReturn(lfs)
      .when(locService).getLocalFileContext(isA(Configuration.class));
  try {
    dispatcher.start();

    // initialize ResourceLocalizationService
    locService.init(conf);

    final FsPermission defaultPerm = new FsPermission((short)0755);

    // verify directory creation
    for (Path p : localDirs) {
      p = new Path((new URI(p.toString())).getPath());
      Path usercache = new Path(p, ContainerLocalizer.USERCACHE);
      verify(spylfs)
          .rename(eq(usercache), any(Path.class), any(Options.Rename.class));
      verify(spylfs)
          .mkdir(eq(usercache),
              eq(defaultPerm), eq(true));
      Path publicCache = new Path(p, ContainerLocalizer.FILECACHE);
      verify(spylfs)
          .rename(eq(usercache), any(Path.class), any(Options.Rename.class));
      verify(spylfs)
          .mkdir(eq(publicCache),
              eq(defaultPerm), eq(true));
      Path nmPriv = new Path(p, ResourceLocalizationService.NM_PRIVATE_DIR);
      verify(spylfs)
          .rename(eq(usercache), any(Path.class), any(Options.Rename.class));
      verify(spylfs).mkdir(eq(nmPriv),
          eq(ResourceLocalizationService.NM_PRIVATE_PERM), eq(true));
    }
  } finally {
    dispatcher.stop();
    delService.stop();
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入方法依赖的package包/类
@Test
public void testLocalizationInit() throws Exception {
  conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(new Configuration());

  ContainerExecutor exec = mock(ContainerExecutor.class);
  DeletionService delService = spy(new DeletionService(exec));
  delService.init(conf);
  delService.start();

  List<Path> localDirs = new ArrayList<Path>();
  String[] sDirs = new String[4];
  for (int i = 0; i < 4; ++i) {
    localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
    sDirs[i] = localDirs.get(i).toString();
  }
  conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);

  LocalDirsHandlerService diskhandler = new LocalDirsHandlerService();
  diskhandler.init(conf);

  ResourceLocalizationService locService =
    spy(new ResourceLocalizationService(dispatcher, exec, delService,
                                        diskhandler));
  doReturn(lfs)
    .when(locService).getLocalFileContext(isA(Configuration.class));
  try {
    dispatcher.start();

    // initialize ResourceLocalizationService
    locService.init(conf);

    final FsPermission defaultPerm = new FsPermission((short)0755);

    // verify directory creation
    for (Path p : localDirs) {
      p = new Path((new URI(p.toString())).getPath());
      Path usercache = new Path(p, ContainerLocalizer.USERCACHE);
      verify(spylfs)
        .mkdir(eq(usercache),
            eq(defaultPerm), eq(true));
      Path publicCache = new Path(p, ContainerLocalizer.FILECACHE);
      verify(spylfs)
        .mkdir(eq(publicCache),
            eq(defaultPerm), eq(true));
      Path nmPriv = new Path(p, ResourceLocalizationService.NM_PRIVATE_DIR);
      verify(spylfs).mkdir(eq(nmPriv),
          eq(ResourceLocalizationService.NM_PRIVATE_PERM), eq(true));
    }
  } finally {
    dispatcher.stop();
    delService.stop();
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入方法依赖的package包/类
@Test
public void testLocalizationInit() throws Exception {
  conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(new Configuration());

  ContainerExecutor exec = mock(ContainerExecutor.class);
  DeletionService delService = spy(new DeletionService(exec));
  delService.init(conf);
  delService.start();

  List<Path> localDirs = new ArrayList<Path>();
  String[] sDirs = new String[4];
  for (int i = 0; i < 4; ++i) {
    localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
    sDirs[i] = localDirs.get(i).toString();
  }
  conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);

  LocalDirsHandlerService diskhandler = new LocalDirsHandlerService();
  diskhandler.init(conf);

  ResourceLocalizationService locService =
    spy(new ResourceLocalizationService(dispatcher, exec, delService,
                                        diskhandler, nmContext));
  doReturn(lfs)
    .when(locService).getLocalFileContext(isA(Configuration.class));
  try {
    dispatcher.start();

    // initialize ResourceLocalizationService
    locService.init(conf);

    final FsPermission defaultPerm = new FsPermission((short)0755);
    final FsPermission securePerm = new FsPermission((short)0711);

    // verify directory creation
    for (Path p : localDirs) {
      p = new Path((new URI(p.toString())).getPath());
      Path usercache = new Path(p, ContainerLocalizer.USERCACHE);
      verify(spylfs)
        .mkdir(eq(usercache),
            eq(securePerm), eq(true));
      Path publicCache = new Path(p, ContainerLocalizer.FILECACHE);
      verify(spylfs)
        .mkdir(eq(publicCache),
            eq(defaultPerm), eq(true));
      Path nmPriv = new Path(p, ResourceLocalizationService.NM_PRIVATE_DIR);
      verify(spylfs).mkdir(eq(nmPriv),
          eq(ResourceLocalizationService.NM_PRIVATE_PERM), eq(true));
    }
  } finally {
    dispatcher.stop();
    delService.stop();
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入方法依赖的package包/类
@Test
public void testDirectoryCleanupOnNewlyCreatedStateStore()
    throws IOException, URISyntaxException {
  conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(new Configuration());

  ContainerExecutor exec = mock(ContainerExecutor.class);
  DeletionService delService = spy(new DeletionService(exec));
  delService.init(conf);
  delService.start();

  List<Path> localDirs = new ArrayList<Path>();
  String[] sDirs = new String[4];
  for (int i = 0; i < 4; ++i) {
    localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
    sDirs[i] = localDirs.get(i).toString();
  }
  conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);

  LocalDirsHandlerService diskhandler = new LocalDirsHandlerService();
  diskhandler.init(conf);

  NMStateStoreService nmStateStoreService = mock(NMStateStoreService.class);
  when(nmStateStoreService.canRecover()).thenReturn(true);
  when(nmStateStoreService.isNewlyCreated()).thenReturn(true);

  ResourceLocalizationService locService =
      spy(new ResourceLocalizationService(dispatcher, exec, delService,
          diskhandler,nmContext));
  doReturn(lfs)
      .when(locService).getLocalFileContext(isA(Configuration.class));
  try {
    dispatcher.start();

    // initialize ResourceLocalizationService
    locService.init(conf);

    final FsPermission defaultPerm = new FsPermission((short)0755);
    final FsPermission securePerm = new FsPermission((short)0711);
    
    // verify directory creation
    for (Path p : localDirs) {
      p = new Path((new URI(p.toString())).getPath());
      Path usercache = new Path(p, ContainerLocalizer.USERCACHE);
      verify(spylfs)
          .rename(eq(usercache), any(Path.class), any(Options.Rename.class));
      verify(spylfs)
          .mkdir(eq(usercache),
              eq(securePerm), eq(true));
      Path publicCache = new Path(p, ContainerLocalizer.FILECACHE);
      verify(spylfs)
          .rename(eq(usercache), any(Path.class), any(Options.Rename.class));
      verify(spylfs)
          .mkdir(eq(publicCache),
              eq(defaultPerm), eq(true));
      Path nmPriv = new Path(p, ResourceLocalizationService.NM_PRIVATE_DIR);
      verify(spylfs)
          .rename(eq(usercache), any(Path.class), any(Options.Rename.class));
      verify(spylfs).mkdir(eq(nmPriv),
          eq(ResourceLocalizationService.NM_PRIVATE_PERM), eq(true));
    }
  } finally {
    dispatcher.stop();
    delService.stop();
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.DeletionService; //导入方法依赖的package包/类
@Test( timeout = 10000)
@SuppressWarnings("unchecked") // mocked generics
public void testLocalizerRunnerException() throws Exception {
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
  dispatcher.register(ApplicationEventType.class, applicationBus);
  EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
  dispatcher.register(ContainerEventType.class, containerBus);

  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
  LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler);
  dirsHandlerSpy.init(conf);

  DeletionService delServiceReal = new DeletionService(exec);
  DeletionService delService = spy(delServiceReal);
  delService.init(new Configuration());
  delService.start();

  ResourceLocalizationService rawService =
      new ResourceLocalizationService(dispatcher, exec, delService,
      dirsHandlerSpy, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  try {
    spyService.init(conf);
    spyService.start();

    // init application
    final Application app = mock(Application.class);
    final ApplicationId appId =
        BuilderUtils.newApplicationId(314159265358979L, 3);
    when(app.getUser()).thenReturn("user0");
    when(app.getAppId()).thenReturn(appId);
    spyService.handle(new ApplicationLocalizationEvent(
        LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
    dispatcher.await();

    Random r = new Random();
    long seed = r.nextLong();
    System.out.println("SEED: " + seed);
    r.setSeed(seed);
    final Container c = getMockContainer(appId, 42, "user0", "user0Folder");
    final LocalResource resource1 = getPrivateMockedResource(r);
    System.out.println("Here 4");
    
    final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
    Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
      new HashMap<LocalResourceVisibility, 
                  Collection<LocalResourceRequest>>();
    List<LocalResourceRequest> privateResourceList =
        new ArrayList<LocalResourceRequest>();
    privateResourceList.add(req1);
    rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);

    final Constructor<?>[] constructors =
        FSError.class.getDeclaredConstructors();
    constructors[0].setAccessible(true);
    FSError fsError =
        (FSError) constructors[0].newInstance(new IOException("Disk Error"));

    Mockito
      .doThrow(fsError)
      .when(dirsHandlerSpy)
      .getLocalPathForWrite(isA(String.class));
    spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
    Thread.sleep(1000);
    dispatcher.await();
    // Verify if ContainerResourceFailedEvent is invoked on FSError
    verify(containerBus).handle(isA(ContainerResourceFailedEvent.class));
  } finally {
    spyService.stop();
    dispatcher.stop();
    delService.stop();
  }
}