Java AsyncDispatcher类代码示例

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入依赖的package包/类
private HashMap<String, String> addAppContainers(Application app) 
    throws IOException {
  Dispatcher dispatcher = new AsyncDispatcher();
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      app.getAppId(), 1);
  Container container1 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 1);
  Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 2);
  nmContext.getContainers()
      .put(container1.getContainerId(), container1);
  nmContext.getContainers()
      .put(container2.getContainerId(), container2);

  app.getContainers().put(container1.getContainerId(), container1);
  app.getContainers().put(container2.getContainerId(), container2);
  HashMap<String, String> hash = new HashMap<String, String>();
  hash.put(container1.getContainerId().toString(), container1
      .getContainerId().toString());
  hash.put(container2.getContainerId().toString(), container2
      .getContainerId().toString());
  return hash;
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入依赖的package包/类
@Before
public void setUp() throws Exception {
  counter = new AtomicInteger(0);
  conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
      "kerberos");
  UserGroupInformation.setConfiguration(conf);
  eventQueue = new LinkedBlockingQueue<Event>();
  dispatcher = new AsyncDispatcher(eventQueue);
  Renewer.reset();
  delegationTokenRenewer = createNewDelegationTokenRenewer(conf, counter);
  RMContext mockContext =  mock(RMContext.class);
  ClientRMService mockClientRMService = mock(ClientRMService.class);
  when(mockContext.getSystemCredentialsForApps()).thenReturn(
    new ConcurrentHashMap<ApplicationId, ByteBuffer>());
  when(mockContext.getDelegationTokenRenewer()).thenReturn(
      delegationTokenRenewer);
  when(mockContext.getDispatcher()).thenReturn(dispatcher);
  when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
  InetSocketAddress sockAddr =
      InetSocketAddress.createUnresolved("localhost", 1234);
  when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
  delegationTokenRenewer.setRMContext(mockContext);
  delegationTokenRenewer.init(conf);
  delegationTokenRenewer.start();
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入依赖的package包/类
@Before
public void setUp() throws IOException {
  scheduler = new FairScheduler();
  
  Configuration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
      ResourceScheduler.class);
  conf.set("yarn.scheduler.fair.event-log-enabled", "true");

  // All tests assume only one assignment per node update
  conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
  resourceManager = new ResourceManager();
  resourceManager.init(conf);
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入依赖的package包/类
@Before
public void setUp() throws IOException {
  scheduler = new FairScheduler();
  conf = createConfiguration();
  resourceManager = new ResourceManager();
  resourceManager.init(conf);

  // TODO: This test should really be using MockRM. For now starting stuff
  // that is needed at a bare minimum.
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  resourceManager.getRMContext().getStateStore().start();

  // to initialize the master key
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();

  scheduler.setRMContext(resourceManager.getRMContext());
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入依赖的package包/类
@Before
public void setUp() throws Exception {
  resourceManager = new ResourceManager() {
    @Override
    protected RMNodeLabelsManager createNodeLabelManager() {
      RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
      mgr.init(getConfig());
      return mgr;
    }
  };
  CapacitySchedulerConfiguration csConf 
     = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, 
      CapacityScheduler.class, ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入依赖的package包/类
@Test(timeout=20000)
public void testCommitJobFailsJob() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  OutputCommitter committer = new TestingOutputCommitter(syncBarrier, false);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, null);
  completeJobTasks(job);
  assertJobState(job, JobStateInternal.COMMITTING);

  // let the committer fail and verify the job fails
  syncBarrier.await();
  assertJobState(job, JobStateInternal.FAILED);
  dispatcher.stop();
  commitHandler.stop();
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入依赖的package包/类
@Test(timeout=20000)
public void testKilledDuringCommit() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  OutputCommitter committer = new WaitingOutputCommitter(syncBarrier, true);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, null);
  completeJobTasks(job);
  assertJobState(job, JobStateInternal.COMMITTING);

  syncBarrier.await();
  job.handle(new JobEvent(job.getID(), JobEventType.JOB_KILL));
  assertJobState(job, JobStateInternal.KILLED);
  dispatcher.stop();
  commitHandler.stop();
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入依赖的package包/类
@Test
public void testTransitionsAtFailed() throws IOException {
  Configuration conf = new Configuration();
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();

  OutputCommitter committer = mock(OutputCommitter.class);
  doThrow(new IOException("forcefail"))
    .when(committer).setupJob(any(JobContext.class));
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  AppContext mockContext = mock(AppContext.class);
  when(mockContext.hasSuccessfullyUnregistered()).thenReturn(false);
  JobImpl job = createStubbedJob(conf, dispatcher, 2, mockContext);
  JobId jobId = job.getID();
  job.handle(new JobEvent(jobId, JobEventType.JOB_INIT));
  assertJobState(job, JobStateInternal.INITED);
  job.handle(new JobStartEvent(jobId));
  assertJobState(job, JobStateInternal.FAILED);

  job.handle(new JobEvent(jobId, JobEventType.JOB_TASK_COMPLETED));
  assertJobState(job, JobStateInternal.FAILED);
  job.handle(new JobEvent(jobId, JobEventType.JOB_TASK_ATTEMPT_COMPLETED));
  assertJobState(job, JobStateInternal.FAILED);
  job.handle(new JobEvent(jobId, JobEventType.JOB_MAP_TASK_RESCHEDULED));
  assertJobState(job, JobStateInternal.FAILED);
  job.handle(new JobEvent(jobId, JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE));
  assertJobState(job, JobStateInternal.FAILED);
  Assert.assertEquals(JobState.RUNNING, job.getState());
  when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true);
  Assert.assertEquals(JobState.FAILED, job.getState());

  dispatcher.stop();
  commitHandler.stop();
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入依赖的package包/类
@Before
public void setUp() {
  this.conf = new YarnConfiguration();
  this.conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
  String mockPassThroughInterceptorClass =
      PassThroughRequestInterceptor.class.getName();

  // Create a request intercepter pipeline for testing. The last one in the
  // chain will call the mock resource manager. The others in the chain will
  // simply forward it to the next one in the chain
  this.conf.set(YarnConfiguration.AMRM_PROXY_INTERCEPTOR_CLASS_PIPELINE,
      mockPassThroughInterceptorClass + ","
          + mockPassThroughInterceptorClass + ","
          + mockPassThroughInterceptorClass + ","
          + MockRequestInterceptor.class.getName());

  this.dispatcher = new AsyncDispatcher();
  this.dispatcher.init(conf);
  this.dispatcher.start();
  this.amrmProxyService = createAndStartAMRMProxyService();
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入依赖的package包/类
@Before
public void setup() {
  executor = new MockExecutor();
  dispatcher = new AsyncDispatcher();
  context = Mockito.mock(Context.class);
  Mockito.doReturn(new ConcurrentSkipListMap<ContainerId, Container>())
      .when(context).getContainers();
  conf = new Configuration();
  conf.set(
      YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR,
      MockResourceCalculatorPlugin.class.getCanonicalName());
  conf.set(
      YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE,
      MockResourceCalculatorProcessTree.class.getCanonicalName());
  dispatcher.init(conf);
  dispatcher.start();
  containerEventHandler = new MockContainerEventHandler();
  dispatcher.register(ContainerEventType.class, containerEventHandler);
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入依赖的package包/类
private Dispatcher getDispatcher() {
  Dispatcher dispatcher = new AsyncDispatcher() {
    @SuppressWarnings({ "rawtypes", "unchecked" })
    @Override
    public EventHandler getEventHandler() {

      class EventArgMatcher extends ArgumentMatcher<AbstractEvent> {
        @Override
        public boolean matches(Object argument) {
          if (argument instanceof RMAppNodeUpdateEvent) {
            ApplicationId appid =
                ((RMAppNodeUpdateEvent) argument).getApplicationId();
            applist.add(appid);
          }
          return false;
        }
      }

      EventHandler handler = spy(super.getEventHandler());
      doNothing().when(handler).handle(argThat(new EventArgMatcher()));
      return handler;
    }
  };
  return dispatcher;
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入依赖的package包/类
public void setup() {
  resourceManager = new ResourceManager() {
    @Override
    protected RMNodeLabelsManager createNodeLabelManager() {
      RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
      mgr.init(getConfig());
      return mgr;
    }
  };

  YarnConfiguration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager()
    .rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start();
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入依赖的package包/类
@Before
public void setUp() throws IOException {
  scheduler = new FairScheduler();
  
  Configuration conf = new YarnConfiguration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
      ResourceScheduler.class);
  conf.set("yarn.scheduler.fair.event-log-enabled", "true");

  // All tests assume only one assignment per node update
  conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
  resourceManager = new MockRM(conf);
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入依赖的package包/类
@Before
public void setUp() throws IOException {
  scheduler = new FairScheduler();
  conf = createConfiguration();
  resourceManager = new MockRM(conf);

  // TODO: This test should really be using MockRM. For now starting stuff
  // that is needed at a bare minimum.
  ((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
  resourceManager.getRMContext().getStateStore().start();

  // to initialize the master key
  resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();

  scheduler.setRMContext(resourceManager.getRMContext());
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入依赖的package包/类
@Before
public void setup() {
  applicationAttemptId = mock(ApplicationAttemptId.class);
  AppContext appContext = mock(AppContext.class);

  dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  JobEventDispatcher jobEventDispatcher = new JobEventDispatcher(appContext);
  TaskEventDispatcher taskEventDispatcher = mock(TaskEventDispatcher.class);
  doNothing().when(taskEventDispatcher).handle(Matchers.any(TaskEvent.class));
  dispatcher.register(JobEventType.class, jobEventDispatcher);
  dispatcher.register(TaskEventType.class, taskEventDispatcher);
  when(applicationAttemptId.toString()).thenReturn("appattempt_1465186316357_0001_000001");
  ApplicationId applicationId = mock(ApplicationId.class);
  when(applicationId.toString()).thenReturn("application_1465186316357_0001");
  when(applicationAttemptId.getApplicationId()).thenReturn(applicationId);
  when(appContext.getEventHandler()).thenReturn(dispatcher.getEventHandler());
  nJob = new NJobImpl("Yacop job", applicationAttemptId, conf, appContext.getEventHandler());
  when(appContext.getJob()).thenReturn(nJob);
}