Java LogHandlerContainerFinishedEvent类代码示例

import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; //导入依赖的package包/类
@Test(timeout = 50000)
public void testLogAggregationAbsentContainer() throws Exception {
  ApplicationId appId = createApplication();
  LogAggregationService logAggregationService =
      createLogAggregationService(appId,
          FailedOrKilledContainerLogAggregationPolicy.class, null);
  ApplicationAttemptId appAttemptId1 =
      BuilderUtils.newApplicationAttemptId(appId, 1);
  ContainerId containerId = BuilderUtils.newContainerId(appAttemptId1, 2l);
  try {
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(
        containerId, 100));
    assertTrue("Should skip when null containerID", true);
  } catch (Exception e) {
    Assert.assertFalse("Exception not expected should skip null containerid",
        true);
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; //导入依赖的package包/类
private ContainerId finishContainer(ApplicationId application1,
    LogAggregationService logAggregationService, ContainerType containerType,
    long cId, int exitCode, String[] logFiles) throws IOException {
  ApplicationAttemptId appAttemptId1 =
      BuilderUtils.newApplicationAttemptId(application1, 1);
  ContainerId containerId = createContainer(appAttemptId1, cId,
      containerType);
  // Simulate log-file creation
  File appLogDir1 =
      new File(localLogDir, ConverterUtils.toString(application1));
  appLogDir1.mkdir();
  writeContainerLogs(appLogDir1, containerId, logFiles);

  logAggregationService.handle(new LogHandlerContainerFinishedEvent(
      containerId, exitCode));
  return containerId;

}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; //导入依赖的package包/类
private ContainerId finishContainer(ApplicationId application1,
    LogAggregationService logAggregationService, ContainerType containerType,
    long cId, int exitCode, String[] logFiles) throws IOException {
  ApplicationAttemptId appAttemptId1 =
      BuilderUtils.newApplicationAttemptId(application1, 1);
  ContainerId containerId = createContainer(appAttemptId1, cId,
      containerType);
  // Simulate log-file creation
  File userFold = new File(localLogDir, userFolder);
  File appLogDir1 = new File(userFold, application1.toString());
  appLogDir1.mkdir();
  writeContainerLogs(appLogDir1, containerId, logFiles);

  logAggregationService.handle(new LogHandlerContainerFinishedEvent(
      containerId, exitCode));
  return containerId;

}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; //导入依赖的package包/类
@Override
public void handle(LogHandlerEvent event) {
  switch (event.getType()) {
    case APPLICATION_STARTED:
      LogHandlerAppStartedEvent appStartEvent =
          (LogHandlerAppStartedEvent) event;
      initApp(appStartEvent.getApplicationId(), appStartEvent.getUser(),
          appStartEvent.getCredentials(),
          appStartEvent.getLogRetentionPolicy(),
          appStartEvent.getApplicationAcls(),
          appStartEvent.getLogAggregationContext());
      break;
    case CONTAINER_FINISHED:
      LogHandlerContainerFinishedEvent containerFinishEvent =
          (LogHandlerContainerFinishedEvent) event;
      stopContainer(containerFinishEvent.getContainerId(),
          containerFinishEvent.getExitCode());
      break;
    case APPLICATION_FINISHED:
      LogHandlerAppFinishedEvent appFinishedEvent =
          (LogHandlerAppFinishedEvent) event;
      stopApp(appFinishedEvent.getApplicationId());
      break;
    default:
      ; // Ignore
  }

}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; //导入依赖的package包/类
@SuppressWarnings("unchecked")
private void sendFinishedEvents() {
  // Inform the application
  @SuppressWarnings("rawtypes")
  EventHandler eventHandler = dispatcher.getEventHandler();
  eventHandler.handle(new ApplicationContainerFinishedEvent(containerId));
  // Remove the container from the resource-monitor
  eventHandler.handle(new ContainerStopMonitoringEvent(containerId));
  // Tell the logService too
  eventHandler.handle(new LogHandlerContainerFinishedEvent(
    containerId, exitCode));
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; //导入依赖的package包/类
@Override
public void handle(LogHandlerEvent event) {
  switch (event.getType()) {
    case APPLICATION_STARTED:
      LogHandlerAppStartedEvent appStartEvent =
          (LogHandlerAppStartedEvent) event;
      initApp(appStartEvent.getApplicationId(), appStartEvent.getUser(),
          appStartEvent.getCredentials(),
          appStartEvent.getApplicationAcls(),
          appStartEvent.getLogAggregationContext());
      break;
    case CONTAINER_FINISHED:
      LogHandlerContainerFinishedEvent containerFinishEvent =
          (LogHandlerContainerFinishedEvent) event;
      stopContainer(containerFinishEvent.getContainerId(),
          containerFinishEvent.getExitCode());
      break;
    case APPLICATION_FINISHED:
      LogHandlerAppFinishedEvent appFinishedEvent =
          (LogHandlerAppFinishedEvent) event;
      stopApp(appFinishedEvent.getApplicationId());
      break;
    default:
      ; // Ignore
  }

}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; //导入依赖的package包/类
@Override
public void handle(LogHandlerEvent event) {
  switch (event.getType()) {
    case APPLICATION_STARTED:
      LogHandlerAppStartedEvent appStartEvent =
          (LogHandlerAppStartedEvent) event;
      initApp(appStartEvent.getApplicationId(), appStartEvent.getUser(),
          appStartEvent.getCredentials(),
          appStartEvent.getLogRetentionPolicy(),
          appStartEvent.getApplicationAcls());
      break;
    case CONTAINER_FINISHED:
      LogHandlerContainerFinishedEvent containerFinishEvent =
          (LogHandlerContainerFinishedEvent) event;
      stopContainer(containerFinishEvent.getContainerId(),
          containerFinishEvent.getExitCode());
      break;
    case APPLICATION_FINISHED:
      LogHandlerAppFinishedEvent appFinishedEvent =
          (LogHandlerAppFinishedEvent) event;
      stopApp(appFinishedEvent.getApplicationId());
      break;
    default:
      ; // Ignore
  }

}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; //导入依赖的package包/类
@Override
public void handle(LogHandlerEvent event) {
  switch (event.getType()) {
    case APPLICATION_STARTED:
      LogHandlerAppStartedEvent appStartEvent =
          (LogHandlerAppStartedEvent) event;
      initApp(appStartEvent.getApplicationId(), appStartEvent.getUser(),
          appStartEvent.getCredentials(),
          appStartEvent.getApplicationAcls(),
          appStartEvent.getLogAggregationContext(),
          appStartEvent.getUserFolder());
      break;
    case CONTAINER_FINISHED:
      LogHandlerContainerFinishedEvent containerFinishEvent =
          (LogHandlerContainerFinishedEvent) event;
      stopContainer(containerFinishEvent.getContainerId(),
          containerFinishEvent.getExitCode());
      break;
    case APPLICATION_FINISHED:
      LogHandlerAppFinishedEvent appFinishedEvent =
          (LogHandlerAppFinishedEvent) event;
      stopApp(appFinishedEvent.getApplicationId());
      break;
    default:
      ; // Ignore
  }

}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; //导入依赖的package包/类
@Test
public void testLogDeletion() throws IOException {
  File[] localLogDirs = getLocalLogDirFiles(this.getClass().getName(), 2);
  String localLogDirsString =
      localLogDirs[0].getAbsolutePath() + ","
          + localLogDirs[1].getAbsolutePath();

  conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDirsString);
  conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, false);
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 0l);

  dirsHandler.init(conf);

  NonAggregatingLogHandler rawLogHandler =
      new NonAggregatingLogHandler(dispatcher, mockDelService, dirsHandler,
          new NMNullStateStoreService());
  NonAggregatingLogHandler logHandler = spy(rawLogHandler);
  AbstractFileSystem spylfs =
      spy(FileContext.getLocalFSFileContext().getDefaultFileSystem());
  FileContext lfs = FileContext.getFileContext(spylfs, conf);
  doReturn(lfs).when(logHandler)
    .getLocalFileContext(isA(Configuration.class));
  FsPermission defaultPermission =
      FsPermission.getDirDefault().applyUMask(lfs.getUMask());
  final FileStatus fs =
      new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0,
        defaultPermission, "", "",
        new Path(localLogDirs[0].getAbsolutePath()));
  doReturn(fs).when(spylfs).getFileStatus(isA(Path.class));

  logHandler.init(conf);
  logHandler.start();

  logHandler.handle(new LogHandlerAppStartedEvent(appId, user, null,
      ContainerLogsRetentionPolicy.ALL_CONTAINERS, null));

  logHandler.handle(new LogHandlerContainerFinishedEvent(container11, 0));

  logHandler.handle(new LogHandlerAppFinishedEvent(appId));

  Path[] localAppLogDirs = new Path[2];
  localAppLogDirs[0] =
      new Path(localLogDirs[0].getAbsolutePath(), appId.toString());
  localAppLogDirs[1] =
      new Path(localLogDirs[1].getAbsolutePath(), appId.toString());

  testDeletionServiceCall(mockDelService, user, 5000, localAppLogDirs);
  logHandler.close();
  for (int i = 0; i < localLogDirs.length; i++) {
    FileUtils.deleteDirectory(localLogDirs[i]);
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; //导入依赖的package包/类
@Test
public void testDelayedDelete() throws IOException {
  File[] localLogDirs = getLocalLogDirFiles(this.getClass().getName(), 2);
  String localLogDirsString =
      localLogDirs[0].getAbsolutePath() + ","
          + localLogDirs[1].getAbsolutePath();

  conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDirsString);
  conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, false);

  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS,
          YarnConfiguration.DEFAULT_NM_LOG_RETAIN_SECONDS);

  dirsHandler.init(conf);

  NonAggregatingLogHandler logHandler =
      new NonAggregatingLogHandlerWithMockExecutor(dispatcher, mockDelService,
                                                   dirsHandler);
  logHandler.init(conf);
  logHandler.start();

  logHandler.handle(new LogHandlerAppStartedEvent(appId, user, null,
      ContainerLogsRetentionPolicy.ALL_CONTAINERS, null));

  logHandler.handle(new LogHandlerContainerFinishedEvent(container11, 0));

  logHandler.handle(new LogHandlerAppFinishedEvent(appId));

  Path[] localAppLogDirs = new Path[2];
  localAppLogDirs[0] =
      new Path(localLogDirs[0].getAbsolutePath(), appId.toString());
  localAppLogDirs[1] =
      new Path(localLogDirs[1].getAbsolutePath(), appId.toString());

  ScheduledThreadPoolExecutor mockSched =
      ((NonAggregatingLogHandlerWithMockExecutor) logHandler).mockSched;

  verify(mockSched).schedule(any(Runnable.class), eq(10800l),
      eq(TimeUnit.SECONDS));
  logHandler.close();
  for (int i = 0; i < localLogDirs.length; i++) {
    FileUtils.deleteDirectory(localLogDirs[i]);
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; //导入依赖的package包/类
@Test
public void testRecovery() throws Exception {
  File[] localLogDirs = getLocalLogDirFiles(this.getClass().getName(), 2);
  String localLogDirsString =
      localLogDirs[0].getAbsolutePath() + ","
          + localLogDirs[1].getAbsolutePath();

  conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDirsString);
  conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, false);

  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS,
          YarnConfiguration.DEFAULT_NM_LOG_RETAIN_SECONDS);

  dirsHandler.init(conf);

  NMStateStoreService stateStore = new NMMemoryStateStoreService();
  stateStore.init(conf);
  stateStore.start();
  NonAggregatingLogHandlerWithMockExecutor logHandler =
      new NonAggregatingLogHandlerWithMockExecutor(dispatcher, mockDelService,
                                                   dirsHandler, stateStore);
  logHandler.init(conf);
  logHandler.start();

  logHandler.handle(new LogHandlerAppStartedEvent(appId, user, null,
      ContainerLogsRetentionPolicy.ALL_CONTAINERS, null));
  logHandler.handle(new LogHandlerContainerFinishedEvent(container11, 0));
  logHandler.handle(new LogHandlerAppFinishedEvent(appId));

  // simulate a restart and verify deletion is rescheduled
  logHandler.close();
  logHandler = new NonAggregatingLogHandlerWithMockExecutor(dispatcher,
      mockDelService, dirsHandler, stateStore);
  logHandler.init(conf);
  logHandler.start();
  ArgumentCaptor<Runnable> schedArg = ArgumentCaptor.forClass(Runnable.class);
  verify(logHandler.mockSched).schedule(schedArg.capture(),
      anyLong(), eq(TimeUnit.MILLISECONDS));

  // execute the runnable and verify another restart has nothing scheduled
  schedArg.getValue().run();
  logHandler.close();
  logHandler = new NonAggregatingLogHandlerWithMockExecutor(dispatcher,
      mockDelService, dirsHandler, stateStore);
  logHandler.init(conf);
  logHandler.start();
  verify(logHandler.mockSched, never()).schedule(any(Runnable.class),
      anyLong(), any(TimeUnit.class));
  logHandler.close();
 }
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; //导入依赖的package包/类
private void verifyLocalFileDeletion(
    LogAggregationService logAggregationService) throws Exception {
  logAggregationService.init(this.conf);
  logAggregationService.start();

  ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);

  // AppLogDir should be created
  File app1LogDir =
      new File(localLogDir, ConverterUtils.toString(application1));
  app1LogDir.mkdir();
  logAggregationService
      .handle(new LogHandlerAppStartedEvent(
          application1, this.user, null,
          ContainerLogsRetentionPolicy.ALL_CONTAINERS, this.acls));

  ApplicationAttemptId appAttemptId =
      BuilderUtils.newApplicationAttemptId(application1, 1);
  ContainerId container11 = BuilderUtils.newContainerId(appAttemptId, 1);
  // Simulate log-file creation
  writeContainerLogs(app1LogDir, container11, new String[] { "stdout",
      "stderr", "syslog" });
  logAggregationService.handle(
      new LogHandlerContainerFinishedEvent(container11, 0));

  logAggregationService.handle(new LogHandlerAppFinishedEvent(
      application1));

  logAggregationService.stop();
  assertEquals(0, logAggregationService.getNumAggregators());
  // ensure filesystems were closed
  verify(logAggregationService).closeFileSystems(
      any(UserGroupInformation.class));
  verify(delSrvc).delete(eq(user), eq((Path) null),
    eq(new Path(app1LogDir.getAbsolutePath())));
  delSrvc.stop();
  
  String containerIdStr = ConverterUtils.toString(container11);
  File containerLogDir = new File(app1LogDir, containerIdStr);
  for (String fileType : new String[] { "stdout", "stderr", "syslog" }) {
    File f = new File(containerLogDir, fileType);
    Assert.assertFalse("check "+f, f.exists());
  }

  Assert.assertFalse(app1LogDir.exists());

  Path logFilePath =
      logAggregationService.getRemoteNodeLogFileForApp(application1,
          this.user);

  Assert.assertTrue("Log file [" + logFilePath + "] not found", new File(
      logFilePath.toUri().getPath()).exists());
  
  dispatcher.await();
  
  ApplicationEvent expectedEvents[] = new ApplicationEvent[]{
      new ApplicationEvent(
          appAttemptId.getApplicationId(),
          ApplicationEventType.APPLICATION_LOG_HANDLING_INITED),
      new ApplicationEvent(
          appAttemptId.getApplicationId(),
          ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED)
  };

  checkEvents(appEventHandler, expectedEvents, true, "getType",
      "getApplicationID");
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; //导入依赖的package包/类
@Test
@SuppressWarnings("unchecked")
public void testLogAggregationInitAppFailsWithoutKillingNM() throws Exception {

  this.conf.set(YarnConfiguration.NM_LOG_DIRS,
      localLogDir.getAbsolutePath());
  this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
      this.remoteRootLogDir.getAbsolutePath());
  
  LogAggregationService logAggregationService = spy(
      new LogAggregationService(dispatcher, this.context, this.delSrvc,
                                super.dirsHandler));
  logAggregationService.init(this.conf);
  logAggregationService.start();

  ApplicationId appId =
      BuilderUtils.newApplicationId(System.currentTimeMillis(),
        (int) (Math.random() * 1000));
  doThrow(new YarnRuntimeException("KABOOM!"))
    .when(logAggregationService).initAppAggregator(
        eq(appId), eq(user), any(Credentials.class),
        any(ContainerLogsRetentionPolicy.class), anyMap(),
        any(LogAggregationContext.class));

  logAggregationService.handle(new LogHandlerAppStartedEvent(appId,
      this.user, null,
      ContainerLogsRetentionPolicy.AM_AND_FAILED_CONTAINERS_ONLY,
      this.acls));

  dispatcher.await();
  ApplicationEvent expectedEvents[] = new ApplicationEvent[]{
      new ApplicationEvent(appId,
          ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED)
  };
  checkEvents(appEventHandler, expectedEvents, false,
      "getType", "getApplicationID", "getDiagnostic");
  // no filesystems instantiated yet
  verify(logAggregationService, never()).closeFileSystems(
      any(UserGroupInformation.class));

  // verify trying to collect logs for containers/apps we don't know about
  // doesn't blow up and tear down the NM
  logAggregationService.handle(new LogHandlerContainerFinishedEvent(
      BuilderUtils.newContainerId(4, 1, 1, 1), 0));
  dispatcher.await();
  logAggregationService.handle(new LogHandlerAppFinishedEvent(
      BuilderUtils.newApplicationId(1, 5)));
  dispatcher.await();
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; //导入依赖的package包/类
@Test
public void testLogAggregationCreateDirsFailsWithoutKillingNM()
    throws Exception {
  
  this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
  this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
      this.remoteRootLogDir.getAbsolutePath());
      
  LogAggregationService logAggregationService = spy(
      new LogAggregationService(dispatcher, this.context, this.delSrvc,
                                super.dirsHandler));
  logAggregationService.init(this.conf);
  logAggregationService.start();
  
  ApplicationId appId =
      BuilderUtils.newApplicationId(System.currentTimeMillis(),
        (int) (Math.random() * 1000));
  Exception e = new RuntimeException("KABOOM!");
  doThrow(e)
    .when(logAggregationService).createAppDir(any(String.class),
        any(ApplicationId.class), any(UserGroupInformation.class));
  logAggregationService.handle(new LogHandlerAppStartedEvent(appId,
      this.user, null,
      ContainerLogsRetentionPolicy.AM_AND_FAILED_CONTAINERS_ONLY, this.acls));        
  
  dispatcher.await();
  ApplicationEvent expectedEvents[] = new ApplicationEvent[]{
      new ApplicationEvent(appId, 
      		ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED)
  };
  checkEvents(appEventHandler, expectedEvents, false,
      "getType", "getApplicationID", "getDiagnostic");
  // filesystems may have been instantiated
  verify(logAggregationService).closeFileSystems(
      any(UserGroupInformation.class));

  // verify trying to collect logs for containers/apps we don't know about
  // doesn't blow up and tear down the NM
  logAggregationService.handle(new LogHandlerContainerFinishedEvent(
      BuilderUtils.newContainerId(4, 1, 1, 1), 0));
  dispatcher.await();
  logAggregationService.handle(new LogHandlerAppFinishedEvent(
      BuilderUtils.newApplicationId(1, 5)));
  dispatcher.await();

  logAggregationService.stop();
  assertEquals(0, logAggregationService.getNumAggregators());
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent; //导入依赖的package包/类
@Test
public void testLogDeletion() throws IOException {
  File[] localLogDirs = getLocalLogDirFiles(this.getClass().getName(), 2);
  String localLogDirsString =
      localLogDirs[0].getAbsolutePath() + ","
          + localLogDirs[1].getAbsolutePath();

  conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDirsString);
  conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, false);
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 0l);

  dirsHandler.init(conf);

  NonAggregatingLogHandler rawLogHandler =
      new NonAggregatingLogHandler(dispatcher, mockDelService, dirsHandler,
          new NMNullStateStoreService());
  NonAggregatingLogHandler logHandler = spy(rawLogHandler);
  AbstractFileSystem spylfs =
      spy(FileContext.getLocalFSFileContext().getDefaultFileSystem());
  FileContext lfs = FileContext.getFileContext(spylfs, conf);
  doReturn(lfs).when(logHandler)
    .getLocalFileContext(isA(Configuration.class));
  FsPermission defaultPermission =
      FsPermission.getDirDefault().applyUMask(lfs.getUMask());
  final FileStatus fs =
      new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0,
        defaultPermission, "", "",
        new Path(localLogDirs[0].getAbsolutePath()));
  doReturn(fs).when(spylfs).getFileStatus(isA(Path.class));

  logHandler.init(conf);
  logHandler.start();

  logHandler.handle(new LogHandlerAppStartedEvent(appId, user, null, null));

  logHandler.handle(new LogHandlerContainerFinishedEvent(container11, 0));

  logHandler.handle(new LogHandlerAppFinishedEvent(appId));

  Path[] localAppLogDirs = new Path[2];
  localAppLogDirs[0] =
      new Path(localLogDirs[0].getAbsolutePath(), appId.toString());
  localAppLogDirs[1] =
      new Path(localLogDirs[1].getAbsolutePath(), appId.toString());

  testDeletionServiceCall(mockDelService, user, 5000, localAppLogDirs);
  logHandler.close();
  for (int i = 0; i < localLogDirs.length; i++) {
    FileUtils.deleteDirectory(localLogDirs[i]);
  }
}