Java ContainerLocalizationRequestEvent类代码示例

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent; //导入依赖的package包/类
/**
 * For each of the requested resources for a container, determines the
 * appropriate {@link LocalResourcesTracker} and forwards a 
 * {@link LocalResourceRequest} to that tracker.
 */
private void handleInitContainerResources(
    ContainerLocalizationRequestEvent rsrcReqs) {
  Container c = rsrcReqs.getContainer();
  // create a loading cache for the file statuses
  LoadingCache<Path,Future<FileStatus>> statCache =
      CacheBuilder.newBuilder().build(FSDownload.createStatusCacheLoader(getConfig()));
  LocalizerContext ctxt = new LocalizerContext(
      c.getUser(), c.getContainerId(), c.getCredentials(), statCache);
  Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
    rsrcReqs.getRequestedResources();
  for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e :
       rsrcs.entrySet()) {
    LocalResourcesTracker tracker =
        getLocalResourcesTracker(e.getKey(), c.getUser(),
            c.getContainerId().getApplicationAttemptId()
                .getApplicationId());
    for (LocalResourceRequest req : e.getValue()) {
      tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
    }
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent; //导入依赖的package包/类
@Override
public boolean matches(Object o) {
  ContainerLocalizationRequestEvent evt =
      (ContainerLocalizationRequestEvent) o;
  final HashSet<LocalResourceRequest> expected =
      new HashSet<LocalResourceRequest>(resources);
  for (Collection<LocalResourceRequest> rc : evt.getRequestedResources()
      .values()) {
    for (LocalResourceRequest rsrc : rc) {
      if (!expected.remove(rsrc)) {
        return false;
      }
    }
  }
  return expected.isEmpty();
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent; //导入依赖的package包/类
/**
 * For each of the requested resources for a container, determines the
 * appropriate {@link LocalResourcesTracker} and forwards a 
 * {@link LocalResourceRequest} to that tracker.
 */
private void handleInitContainerResources(
    ContainerLocalizationRequestEvent rsrcReqs) {
  Container c = rsrcReqs.getContainer();
  // create a loading cache for the file statuses
  LoadingCache<Path,Future<FileStatus>> statCache =
      CacheBuilder.newBuilder().build(FSDownload.createStatusCacheLoader(getConfig()));
  LocalizerContext ctxt = new LocalizerContext(
      c.getUser(), c.getContainerId(), c.getCredentials(), statCache);
  Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
    rsrcReqs.getRequestedResources();
  for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e :
       rsrcs.entrySet()) {
    LocalResourcesTracker tracker =
        getLocalResourcesTracker(e.getKey(), c.getUser(),
            c.getContainerId().getApplicationAttemptId()
                .getApplicationId());
    for (LocalResourceRequest req : e.getValue()) {
      tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
      if (LOG.isDebugEnabled()) {
        LOG.debug("Localizing " + req.getPath() +
            " for container " + c.getContainerId());
      }
    }
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent; //导入依赖的package包/类
@Override
public void handle(LocalizationEvent event) {
  // TODO: create log dir as $logdir/$user/$appId
  switch (event.getType()) {
  case INIT_APPLICATION_RESOURCES:
    handleInitApplicationResources(
        ((ApplicationLocalizationEvent)event).getApplication());
    break;
  case INIT_CONTAINER_RESOURCES:
    handleInitContainerResources((ContainerLocalizationRequestEvent) event);
    break;
  case CACHE_CLEANUP:
    handleCacheCleanup(event);
    break;
  case CLEANUP_CONTAINER_RESOURCES:
    handleCleanupContainerResources((ContainerLocalizationCleanupEvent)event);
    break;
  case DESTROY_APPLICATION_RESOURCES:
    handleDestroyApplicationResources(
        ((ApplicationLocalizationEvent)event).getApplication());
    break;
  default:
    throw new YarnRuntimeException("Unknown localization event: " + event);
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent; //导入依赖的package包/类
/**
 * For each of the requested resources for a container, determines the
 * appropriate {@link LocalResourcesTracker} and forwards a 
 * {@link LocalResourceRequest} to that tracker.
 */
private void handleInitContainerResources(
    ContainerLocalizationRequestEvent rsrcReqs) {
  Container c = rsrcReqs.getContainer();
  LocalizerContext ctxt = new LocalizerContext(
      c.getUser(), c.getContainerId(), c.getCredentials());
  Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
    rsrcReqs.getRequestedResources();
  for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e :
       rsrcs.entrySet()) {
    LocalResourcesTracker tracker =
        getLocalResourcesTracker(e.getKey(), c.getUser(),
            c.getContainerId().getApplicationAttemptId()
                .getApplicationId());
    for (LocalResourceRequest req : e.getValue()) {
      tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
    }
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent; //导入依赖的package包/类
/**
 * For each of the requested resources for a container, determines the
 * appropriate {@link LocalResourcesTracker} and forwards a 
 * {@link LocalResourceRequest} to that tracker.
 */
private void handleInitContainerResources(
    ContainerLocalizationRequestEvent rsrcReqs) {
  Container c = rsrcReqs.getContainer();
  // create a loading cache for the file statuses
  LoadingCache<Path,Future<FileStatus>> statCache =
      CacheBuilder.newBuilder().build(FSDownload.createStatusCacheLoader(getConfig()));
  LocalizerContext ctxt = new LocalizerContext(
      c.getUser(), c.getContainerId(), c.getCredentials(), statCache, c.getUserFolder());
  Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
    rsrcReqs.getRequestedResources();
  for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e :
       rsrcs.entrySet()) {
    LocalResourcesTracker tracker =
        getLocalResourcesTracker(e.getKey(), c.getUser(),
            c.getContainerId().getApplicationAttemptId()
                .getApplicationId());
    for (LocalResourceRequest req : e.getValue()) {
      tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
      if (LOG.isDebugEnabled()) {
        LOG.debug("Localizing " + req.getPath() +
            " for container " + c.getContainerId());
      }
    }
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent; //导入依赖的package包/类
@Override
public void handle(LocalizationEvent event) {
  // TODO: create log dir as $logdir/$user/$appId
  switch (event.getType()) {
  case INIT_APPLICATION_RESOURCES:
    handleInitApplicationResources(
        ((ApplicationLocalizationEvent)event).getApplication());
    break;
  case INIT_CONTAINER_RESOURCES:
    handleInitContainerResources((ContainerLocalizationRequestEvent) event);
    break;
  case CONTAINER_RESOURCES_LOCALIZED:
    handleContainerResourcesLocalized((ContainerLocalizationEvent) event);
    break;
  case CACHE_CLEANUP:
    handleCacheCleanup(event);
    break;
  case CLEANUP_CONTAINER_RESOURCES:
    handleCleanupContainerResources((ContainerLocalizationCleanupEvent)event);
    break;
  case DESTROY_APPLICATION_RESOURCES:
    handleDestroyApplicationResources(
        ((ApplicationLocalizationEvent)event).getApplication());
    break;
  default:
    throw new YarnRuntimeException("Unknown localization event: " + event);
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent; //导入依赖的package包/类
private ContainerLocalizationRequestEvent createContainerLocalizationEvent(
    ContainerImpl container, LocalResourceVisibility vis,
    LocalResourceRequest req) {
  Map<LocalResourceVisibility, Collection<LocalResourceRequest>> reqs =
      new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
  List<LocalResourceRequest> resourceList =
      new ArrayList<LocalResourceRequest>();
  resourceList.add(req);
  reqs.put(vis, resourceList);
  return new ContainerLocalizationRequestEvent(container, reqs);
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent; //导入依赖的package包/类
@Override
public void handle(LocalizationEvent event) {
  // TODO: create log dir as $logdir/$user/$appId
  switch (event.getType()) {
  case INIT_APPLICATION_RESOURCES:
    handleInitApplicationResources(
        ((ApplicationLocalizationEvent)event).getApplication());
    break;
  case INIT_CONTAINER_RESOURCES:
    handleInitContainerResources((ContainerLocalizationRequestEvent) event);
    break;
  case CONTAINER_RESOURCES_LOCALIZED:
    handleContainerResourcesLocalized((ContainerLocalizationEvent) event);
    break;
  case CACHE_CLEANUP:
    handleCacheCleanup();
    break;
  case CLEANUP_CONTAINER_RESOURCES:
    handleCleanupContainerResources((ContainerLocalizationCleanupEvent)event);
    break;
  case DESTROY_APPLICATION_RESOURCES:
    handleDestroyApplicationResources(
        ((ApplicationLocalizationEvent)event).getApplication());
    break;
  default:
    throw new YarnRuntimeException("Unknown localization event: " + event);
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent; //导入依赖的package包/类
@Test( timeout = 10000)
@SuppressWarnings("unchecked") // mocked generics
public void testLocalizerRunnerException() throws Exception {
  DrainDispatcher dispatcher = new DrainDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
  dispatcher.register(ApplicationEventType.class, applicationBus);
  EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
  dispatcher.register(ContainerEventType.class, containerBus);

  ContainerExecutor exec = mock(ContainerExecutor.class);
  LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
  LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler);
  dirsHandlerSpy.init(conf);

  DeletionService delServiceReal = new DeletionService(exec);
  DeletionService delService = spy(delServiceReal);
  delService.init(new Configuration());
  delService.start();

  ResourceLocalizationService rawService =
      new ResourceLocalizationService(dispatcher, exec, delService,
      dirsHandlerSpy, nmContext);
  ResourceLocalizationService spyService = spy(rawService);
  doReturn(mockServer).when(spyService).createServer();
  try {
    spyService.init(conf);
    spyService.start();

    // init application
    final Application app = mock(Application.class);
    final ApplicationId appId =
        BuilderUtils.newApplicationId(314159265358979L, 3);
    when(app.getUser()).thenReturn("user0");
    when(app.getAppId()).thenReturn(appId);
    spyService.handle(new ApplicationLocalizationEvent(
        LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
    dispatcher.await();

    Random r = new Random();
    long seed = r.nextLong();
    System.out.println("SEED: " + seed);
    r.setSeed(seed);
    final Container c = getMockContainer(appId, 42, "user0");
    final LocalResource resource1 = getPrivateMockedResource(r);
    System.out.println("Here 4");
    
    final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
    Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
      new HashMap<LocalResourceVisibility, 
                  Collection<LocalResourceRequest>>();
    List<LocalResourceRequest> privateResourceList =
        new ArrayList<LocalResourceRequest>();
    privateResourceList.add(req1);
    rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);

    final Constructor<?>[] constructors =
        FSError.class.getDeclaredConstructors();
    constructors[0].setAccessible(true);
    FSError fsError =
        (FSError) constructors[0].newInstance(new IOException("Disk Error"));

    Mockito
      .doThrow(fsError)
      .when(dirsHandlerSpy)
      .getLocalPathForWrite(isA(String.class));
    spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
    Thread.sleep(1000);
    dispatcher.await();
    // Verify if ContainerResourceFailedEvent is invoked on FSError
    verify(containerBus).handle(isA(ContainerResourceFailedEvent.class));
  } finally {
    spyService.stop();
    dispatcher.stop();
    delService.stop();
  }
}
 

import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent; //导入依赖的package包/类
@Override
@SuppressWarnings("unchecked")
protected ResourceLocalizationService createResourceLocalizationService(
    ContainerExecutor exec, DeletionService deletionContext, Context context) {
  return new ResourceLocalizationService(super.dispatcher, exec,
      deletionContext, super.dirsHandler, context) {
    @Override
    public void handle(LocalizationEvent event) {
      switch (event.getType()) {
      case INIT_APPLICATION_RESOURCES:
        Application app =
            ((ApplicationLocalizationEvent) event).getApplication();
        // Simulate event from ApplicationLocalization.
        dispatcher.getEventHandler().handle(new ApplicationInitedEvent(
              app.getAppId()));
        break;
      case INIT_CONTAINER_RESOURCES:
        ContainerLocalizationRequestEvent rsrcReqs =
          (ContainerLocalizationRequestEvent) event;
        // simulate localization of all requested resources
          for (Collection<LocalResourceRequest> rc : rsrcReqs
              .getRequestedResources().values()) {
            for (LocalResourceRequest req : rc) {
              LOG.info("DEBUG: " + req + ":"
                  + rsrcReqs.getContainer().getContainerId());
              dispatcher.getEventHandler().handle(
                  new ContainerResourceLocalizedEvent(rsrcReqs.getContainer()
                      .getContainerId(), req, new Path("file:///local"
                      + req.getPath().toUri().getPath())));
            }
          }
        break;
      case CLEANUP_CONTAINER_RESOURCES:
        Container container =
            ((ContainerLocalizationEvent) event).getContainer();
        // TODO: delete the container dir
        this.dispatcher.getEventHandler().handle(
            new ContainerEvent(container.getContainerId(),
                ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP));
        break;
      case DESTROY_APPLICATION_RESOURCES:
        Application application =
          ((ApplicationLocalizationEvent) event).getApplication();

        // decrement reference counts of all resources associated with this
        // app
        this.dispatcher.getEventHandler().handle(
            new ApplicationEvent(application.getAppId(),
                ApplicationEventType.APPLICATION_RESOURCES_CLEANEDUP));
        break;
      default:
        fail("Unexpected event: " + event.getType());
      }
    }
  };
}