Java ContainerResourceFailedEvent类代码示例

import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent; //导入依赖的package包/类
@Override
public void transition(LocalizedResource rsrc, ResourceEvent event) {
  ResourceFailedLocalizationEvent failedEvent =
      (ResourceFailedLocalizationEvent) event;
  Queue<ContainerId> containers = rsrc.ref;
  for (ContainerId container : containers) {
    rsrc.dispatcher.getEventHandler().handle(
      new ContainerResourceFailedEvent(container, failedEvent
        .getLocalResourceRequest(), failedEvent.getDiagnosticMessage()));
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent; //导入依赖的package包/类
@Override
@SuppressWarnings("unchecked") // dispatcher not typed
public void run() {
  Path nmPrivateCTokensPath = null;
  Throwable exception = null;
  try {
    // Get nmPrivateDir
    nmPrivateCTokensPath =
      dirsHandler.getLocalPathForWrite(
            NM_PRIVATE_DIR + Path.SEPARATOR
                + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
                    localizerId));

    // 0) init queue, etc.
    // 1) write credentials to private dir
    writeCredentials(nmPrivateCTokensPath);
    // 2) exec initApplication and wait
    if (dirsHandler.areDisksHealthy()) {
      exec.startLocalizer(new LocalizerStartContext.Builder()
          .setNmPrivateContainerTokens(nmPrivateCTokensPath)
          .setNmAddr(localizationServerAddress)
          .setUser(context.getUser())
          .setAppId(ConverterUtils.toString(context.getContainerId()
              .getApplicationAttemptId().getApplicationId()))
          .setLocId(localizerId)
          .setDirsHandler(dirsHandler)
          .build());
    } else {
      throw new IOException("All disks failed. "
          + dirsHandler.getDisksHealthReport(false));
    }
  // TODO handle ExitCodeException separately?
  } catch (FSError fe) {
    exception = fe;
  } catch (Exception e) {
    exception = e;
  } finally {
    if (exception != null) {
      LOG.info("Localizer failed", exception);
      // On error, report failure to Container and signal ABORT
      // Notify resource of failed localization
      ContainerId cId = context.getContainerId();
      dispatcher.getEventHandler().handle(new ContainerResourceFailedEvent(
          cId, null, exception.getMessage()));
    }
    for (LocalizerResourceRequestEvent event : scheduled.values()) {
      event.getResource().unlock();
    }
    delService.delete(null, nmPrivateCTokensPath, new Path[] {});
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent; //导入依赖的package包/类
@Test( timeout = 10000)
@SuppressWarnings("unchecked") // mocked generics
public void testLocalizerRunnerException() throws Exception {
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
  dispatcher.register(ApplicationEventType.class, applicationBus);
  EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
  dispatcher.register(ContainerEventType.class, containerBus);

  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
  LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler);
  dirsHandlerSpy.init(conf);

  DeletionService delServiceReal = new DeletionService(exec);
  DeletionService delService = spy(delServiceReal);
  delService.init(new Configuration());
  delService.start();

  ResourceLocalizationService rawService =
      new ResourceLocalizationService(dispatcher, exec, delService,
      dirsHandlerSpy, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  try {
    spyService.init(conf);
    spyService.start();

    // init application
    final Application app = mock(Application.class);
    final ApplicationId appId =
        BuilderUtils.newApplicationId(314159265358979L, 3);
    when(app.getUser()).thenReturn("user0");
    when(app.getAppId()).thenReturn(appId);
    spyService.handle(new ApplicationLocalizationEvent(
        LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
    dispatcher.await();

    Random r = new Random();
    long seed = r.nextLong();
    System.out.println("SEED: " + seed);
    r.setSeed(seed);
    final Container c = getMockContainer(appId, 42, "user0");
    final LocalResource resource1 = getPrivateMockedResource(r);
    System.out.println("Here 4");
    
    final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
    Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
      new HashMap<LocalResourceVisibility, 
                  Collection<LocalResourceRequest>>();
    List<LocalResourceRequest> privateResourceList =
        new ArrayList<LocalResourceRequest>();
    privateResourceList.add(req1);
    rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);

    final Constructor<?>[] constructors =
        FSError.class.getDeclaredConstructors();
    constructors[0].setAccessible(true);
    FSError fsError =
        (FSError) constructors[0].newInstance(new IOException("Disk Error"));

    Mockito
      .doThrow(fsError)
      .when(dirsHandlerSpy)
      .getLocalPathForWrite(isA(String.class));
    spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
    Thread.sleep(1000);
    dispatcher.await();
    // Verify if ContainerResourceFailedEvent is invoked on FSError
    verify(containerBus).handle(isA(ContainerResourceFailedEvent.class));
  } finally {
    spyService.stop();
    dispatcher.stop();
    delService.stop();
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent; //导入依赖的package包/类
@Override
@SuppressWarnings("unchecked") // dispatcher not typed
public void run() {
  Path nmPrivateCTokensPath = null;
  Throwable exception = null;
  try {
    // Get nmPrivateDir
    nmPrivateCTokensPath =
      dirsHandler.getLocalPathForWrite(
            NM_PRIVATE_DIR + Path.SEPARATOR
                + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
                    localizerId));

    // 0) init queue, etc.
    // 1) write credentials to private dir
    writeCredentials(nmPrivateCTokensPath);
    // 2) exec initApplication and wait
    if (dirsHandler.areDisksHealthy()) {
      exec.startLocalizer(new LocalizerStartContext.Builder()
          .setNmPrivateContainerTokens(nmPrivateCTokensPath)
          .setNmAddr(localizationServerAddress)
          .setUser(context.getUser())
          .setAppId(ConverterUtils.toString(context.getContainerId()
              .getApplicationAttemptId().getApplicationId()))
          .setLocId(localizerId)
          .setDirsHandler(dirsHandler)
          .build());
    } else {
      throw new IOException("All disks failed. "
          + dirsHandler.getDisksHealthReport(false));
    }
  // TODO handle ExitCodeException separately?
  } catch (FSError fe) {
    exception = fe;
  } catch (Exception e) {
    exception = e;
  } finally {
    if (exception != null) {
      LOG.info("Localizer failed", exception);
      // On error, report failure to Container and signal ABORT
      // Notify resource of failed localization
      ContainerId cId = context.getContainerId();
      dispatcher.getEventHandler().handle(new ContainerResourceFailedEvent(
          cId, null, exception.getMessage()));
    }
    List<Path> paths = new ArrayList<Path>();
    for (LocalizerResourceRequestEvent event : scheduled.values()) {
      // This means some resources were in downloading state. Schedule
      // deletion task for localization dir and tmp dir used for downloading
      Path locRsrcPath = event.getResource().getLocalPath();
      if (locRsrcPath != null) {
        Path locRsrcDirPath = locRsrcPath.getParent();
        paths.add(locRsrcDirPath);
        paths.add(new Path(locRsrcDirPath + "_tmp"));
      }
      event.getResource().unlock();
    }
    if (!paths.isEmpty()) {
      delService.delete(context.getUser(),
          null, paths.toArray(new Path[paths.size()]));
    }
    delService.delete(null, nmPrivateCTokensPath, new Path[] {});
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent; //导入依赖的package包/类
@Override
@SuppressWarnings("unchecked") // dispatcher not typed
public void run() {
  Path nmPrivateCTokensPath = null;
  Throwable exception = null;
  try {
    // Get nmPrivateDir
    nmPrivateCTokensPath =
      dirsHandler.getLocalPathForWrite(
            NM_PRIVATE_DIR + Path.SEPARATOR
                + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
                    localizerId));

    // 0) init queue, etc.
    // 1) write credentials to private dir
    writeCredentials(nmPrivateCTokensPath);
    // 2) exec initApplication and wait
    List<String> localDirs = getInitializedLocalDirs();
    List<String> logDirs = getInitializedLogDirs();
    if (dirsHandler.areDisksHealthy()) {
      exec.startLocalizer(nmPrivateCTokensPath, localizationServerAddress,
          context.getUser(),
          ConverterUtils.toString(
              context.getContainerId().
              getApplicationAttemptId().getApplicationId()),
          localizerId,
          dirsHandler);
    } else {
      throw new IOException("All disks failed. "
          + dirsHandler.getDisksHealthReport(false));
    }
  // TODO handle ExitCodeException separately?
  } catch (FSError fe) {
    exception = fe;
  } catch (Exception e) {
    exception = e;
  } finally {
    if (exception != null) {
      LOG.info("Localizer failed", exception);
      // On error, report failure to Container and signal ABORT
      // Notify resource of failed localization
      ContainerId cId = context.getContainerId();
      dispatcher.getEventHandler().handle(new ContainerResourceFailedEvent(
          cId, null, exception.getMessage()));
    }
    for (LocalizerResourceRequestEvent event : scheduled.values()) {
      event.getResource().unlock();
    }
    delService.delete(null, nmPrivateCTokensPath, new Path[] {});
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent; //导入依赖的package包/类
@Override
@SuppressWarnings("unchecked") // dispatcher not typed
public void run() {
  Path nmPrivateCTokensPath = null;
  Throwable exception = null;
  try {
    // Get nmPrivateDir
    nmPrivateCTokensPath =
      dirsHandler.getLocalPathForWrite(
            NM_PRIVATE_DIR + Path.SEPARATOR
                + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
                    localizerId));

    // 0) init queue, etc.
    // 1) write credentials to private dir
    writeCredentials(nmPrivateCTokensPath);
    // 2) exec initApplication and wait
    if (dirsHandler.areDisksHealthy()) {
      exec.startLocalizer(nmPrivateCTokensPath, localizationServerAddress,
          context.getUser(),
          ConverterUtils.toString(
              context.getContainerId().
              getApplicationAttemptId().getApplicationId()),
          localizerId,
          dirsHandler);
    } else {
      throw new IOException("All disks failed. "
          + dirsHandler.getDisksHealthReport(false));
    }
  // TODO handle ExitCodeException separately?
  } catch (FSError fe) {
    exception = fe;
  } catch (Exception e) {
    exception = e;
  } finally {
    if (exception != null) {
      LOG.info("Localizer failed", exception);
      // On error, report failure to Container and signal ABORT
      // Notify resource of failed localization
      ContainerId cId = context.getContainerId();
      dispatcher.getEventHandler().handle(new ContainerResourceFailedEvent(
          cId, null, exception.getMessage()));
    }
    for (LocalizerResourceRequestEvent event : scheduled.values()) {
      event.getResource().unlock();
    }
    delService.delete(null, nmPrivateCTokensPath, new Path[] {});
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent; //导入依赖的package包/类
@Override
@SuppressWarnings("unchecked") // dispatcher not typed
public void run() {
  Path nmPrivateCTokensPath = null;
  try {
    // Get nmPrivateDir
    nmPrivateCTokensPath =
      dirsHandler.getLocalPathForWrite(
            NM_PRIVATE_DIR + Path.SEPARATOR
                + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
                    localizerId));

    // 0) init queue, etc.
    // 1) write credentials to private dir
    writeCredentials(nmPrivateCTokensPath);
    // 2) exec initApplication and wait
    List<String> localDirs = dirsHandler.getLocalDirs();
    List<String> logDirs = dirsHandler.getLogDirs();
    if (dirsHandler.areDisksHealthy()) {
      exec.startLocalizer(nmPrivateCTokensPath, localizationServerAddress,
          context.getUser(),
          ConverterUtils.toString(
              context.getContainerId().
              getApplicationAttemptId().getApplicationId()),
          localizerId, localDirs, logDirs);
    } else {
      throw new IOException("All disks failed. "
          + dirsHandler.getDisksHealthReport());
    }
  // TODO handle ExitCodeException separately?
  } catch (Exception e) {
    LOG.info("Localizer failed", e);
    // 3) on error, report failure to Container and signal ABORT
    // 3.1) notify resource of failed localization
    ContainerId cId = context.getContainerId();
    dispatcher.getEventHandler().handle(
        new ContainerResourceFailedEvent(cId, null, e.getMessage()));
  } finally {
    for (LocalizerResourceRequestEvent event : scheduled.values()) {
      event.getResource().unlock();
    }
    delService.delete(null, nmPrivateCTokensPath, new Path[] {});
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent; //导入依赖的package包/类
@Override
@SuppressWarnings("unchecked") // dispatcher not typed
public void run() {
  Path nmPrivateCTokensPath = null;
  Throwable exception = null;
  try {
    // Get nmPrivateDir
    nmPrivateCTokensPath =
      dirsHandler.getLocalPathForWrite(
            NM_PRIVATE_DIR + Path.SEPARATOR
                + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
                    localizerId));

    // 0) init queue, etc.
    // 1) write credentials to private dir
    writeCredentials(nmPrivateCTokensPath);
    // 2) exec initApplication and wait
    if (dirsHandler.areDisksHealthy()) {
      exec.startLocalizer(new LocalizerStartContext.Builder()
          .setNmPrivateContainerTokens(nmPrivateCTokensPath)
          .setNmAddr(localizationServerAddress)
          .setUser(context.getUser())
          .setAppId(context.getContainerId()
              .getApplicationAttemptId().getApplicationId().toString())
          .setLocId(localizerId)
          .setDirsHandler(dirsHandler)
          .setUserFolder(context.getUserFolder())
          .build());
    } else {
      throw new IOException("All disks failed. "
          + dirsHandler.getDisksHealthReport(false));
    }
  // TODO handle ExitCodeException separately?
  } catch (FSError fe) {
    exception = fe;
  } catch (Exception e) {
    exception = e;
  } finally {
    if (exception != null) {
      LOG.info("Localizer failed", exception);
      // On error, report failure to Container and signal ABORT
      // Notify resource of failed localization
      ContainerId cId = context.getContainerId();
      dispatcher.getEventHandler().handle(new ContainerResourceFailedEvent(
          cId, null, exception.getMessage()));
    }
    List<Path> paths = new ArrayList<Path>();
    for (LocalizerResourceRequestEvent event : scheduled.values()) {
      // This means some resources were in downloading state. Schedule
      // deletion task for localization dir and tmp dir used for downloading
      Path locRsrcPath = event.getResource().getLocalPath();
      if (locRsrcPath != null) {
        Path locRsrcDirPath = locRsrcPath.getParent();
        paths.add(locRsrcDirPath);
        paths.add(new Path(locRsrcDirPath + "_tmp"));
      }
      event.getResource().unlock();
    }
    if (!paths.isEmpty()) {
      delService.delete(context.getUser(),
          null, paths.toArray(new Path[paths.size()]));
    }
    delService.delete(null, nmPrivateCTokensPath, new Path[] {});
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent; //导入依赖的package包/类
@Test( timeout = 10000)
@SuppressWarnings("unchecked") // mocked generics
public void testLocalizerRunnerException() throws Exception {
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
  dispatcher.register(ApplicationEventType.class, applicationBus);
  EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
  dispatcher.register(ContainerEventType.class, containerBus);

  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
  LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler);
  dirsHandlerSpy.init(conf);

  DeletionService delServiceReal = new DeletionService(exec);
  DeletionService delService = spy(delServiceReal);
  delService.init(new Configuration());
  delService.start();

  ResourceLocalizationService rawService =
      new ResourceLocalizationService(dispatcher, exec, delService,
      dirsHandlerSpy, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  try {
    spyService.init(conf);
    spyService.start();

    // init application
    final Application app = mock(Application.class);
    final ApplicationId appId =
        BuilderUtils.newApplicationId(314159265358979L, 3);
    when(app.getUser()).thenReturn("user0");
    when(app.getAppId()).thenReturn(appId);
    spyService.handle(new ApplicationLocalizationEvent(
        LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
    dispatcher.await();

    Random r = new Random();
    long seed = r.nextLong();
    System.out.println("SEED: " + seed);
    r.setSeed(seed);
    final Container c = getMockContainer(appId, 42, "user0", "user0Folder");
    final LocalResource resource1 = getPrivateMockedResource(r);
    System.out.println("Here 4");
    
    final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
    Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
      new HashMap<LocalResourceVisibility, 
                  Collection<LocalResourceRequest>>();
    List<LocalResourceRequest> privateResourceList =
        new ArrayList<LocalResourceRequest>();
    privateResourceList.add(req1);
    rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);

    final Constructor<?>[] constructors =
        FSError.class.getDeclaredConstructors();
    constructors[0].setAccessible(true);
    FSError fsError =
        (FSError) constructors[0].newInstance(new IOException("Disk Error"));

    Mockito
      .doThrow(fsError)
      .when(dirsHandlerSpy)
      .getLocalPathForWrite(isA(String.class));
    spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
    Thread.sleep(1000);
    dispatcher.await();
    // Verify if ContainerResourceFailedEvent is invoked on FSError
    verify(containerBus).handle(isA(ContainerResourceFailedEvent.class));
  } finally {
    spyService.stop();
    dispatcher.stop();
    delService.stop();
  }
}