Java AsyncDispatcher.init方法代码示例

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入方法依赖的package包/类
@Test(timeout=20000)
public void testCommitJobFailsJob() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  OutputCommitter committer = new TestingOutputCommitter(syncBarrier, false);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, null);
  completeJobTasks(job);
  assertJobState(job, JobStateInternal.COMMITTING);

  // let the committer fail and verify the job fails
  syncBarrier.await();
  assertJobState(job, JobStateInternal.FAILED);
  dispatcher.stop();
  commitHandler.stop();
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入方法依赖的package包/类
@Test(timeout=20000)
public void testKilledDuringCommit() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  OutputCommitter committer = new WaitingOutputCommitter(syncBarrier, true);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, null);
  completeJobTasks(job);
  assertJobState(job, JobStateInternal.COMMITTING);

  syncBarrier.await();
  job.handle(new JobEvent(job.getID(), JobEventType.JOB_KILL));
  assertJobState(job, JobStateInternal.KILLED);
  dispatcher.stop();
  commitHandler.stop();
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入方法依赖的package包/类
@Test
public void testTransitionsAtFailed() throws IOException {
  Configuration conf = new Configuration();
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();

  OutputCommitter committer = mock(OutputCommitter.class);
  doThrow(new IOException("forcefail"))
    .when(committer).setupJob(any(JobContext.class));
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  AppContext mockContext = mock(AppContext.class);
  when(mockContext.hasSuccessfullyUnregistered()).thenReturn(false);
  JobImpl job = createStubbedJob(conf, dispatcher, 2, mockContext);
  JobId jobId = job.getID();
  job.handle(new JobEvent(jobId, JobEventType.JOB_INIT));
  assertJobState(job, JobStateInternal.INITED);
  job.handle(new JobStartEvent(jobId));
  assertJobState(job, JobStateInternal.FAILED);

  job.handle(new JobEvent(jobId, JobEventType.JOB_TASK_COMPLETED));
  assertJobState(job, JobStateInternal.FAILED);
  job.handle(new JobEvent(jobId, JobEventType.JOB_TASK_ATTEMPT_COMPLETED));
  assertJobState(job, JobStateInternal.FAILED);
  job.handle(new JobEvent(jobId, JobEventType.JOB_MAP_TASK_RESCHEDULED));
  assertJobState(job, JobStateInternal.FAILED);
  job.handle(new JobEvent(jobId, JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE));
  assertJobState(job, JobStateInternal.FAILED);
  Assert.assertEquals(JobState.RUNNING, job.getState());
  when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true);
  Assert.assertEquals(JobState.FAILED, job.getState());

  dispatcher.stop();
  commitHandler.stop();
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入方法依赖的package包/类
@Before
public void setup() {
  executor = new MockExecutor();
  dispatcher = new AsyncDispatcher();
  context = Mockito.mock(Context.class);
  Mockito.doReturn(new ConcurrentSkipListMap<ContainerId, Container>())
      .when(context).getContainers();
  conf = new Configuration();
  conf.set(
      YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR,
      MockResourceCalculatorPlugin.class.getCanonicalName());
  conf.set(
      YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE,
      MockResourceCalculatorProcessTree.class.getCanonicalName());
  dispatcher.init(conf);
  dispatcher.start();
  containerEventHandler = new MockContainerEventHandler();
  dispatcher.register(ContainerEventType.class, containerEventHandler);
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入方法依赖的package包/类
@Before
public void setup() {
  applicationAttemptId = mock(ApplicationAttemptId.class);
  AppContext appContext = mock(AppContext.class);

  dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  JobEventDispatcher jobEventDispatcher = new JobEventDispatcher(appContext);
  TaskEventDispatcher taskEventDispatcher = mock(TaskEventDispatcher.class);
  doNothing().when(taskEventDispatcher).handle(Matchers.any(TaskEvent.class));
  dispatcher.register(JobEventType.class, jobEventDispatcher);
  dispatcher.register(TaskEventType.class, taskEventDispatcher);
  when(applicationAttemptId.toString()).thenReturn("appattempt_1465186316357_0001_000001");
  ApplicationId applicationId = mock(ApplicationId.class);
  when(applicationId.toString()).thenReturn("application_1465186316357_0001");
  when(applicationAttemptId.getApplicationId()).thenReturn(applicationId);
  when(appContext.getEventHandler()).thenReturn(dispatcher.getEventHandler());
  nJob = new NJobImpl("Yacop job", applicationAttemptId, conf, appContext.getEventHandler());
  when(appContext.getJob()).thenReturn(nJob);
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入方法依赖的package包/类
@Before
public void setup() {
  YarnConfiguration conf = new YarnConfiguration();
  appContext = mock(AppContext.class);

  dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  taskEventDispatcher = new TaskEventDispatcher(appContext);
  dispatcher.register(TaskEventType.class, taskEventDispatcher);
  containerAllocator = mock(ContainerAllocator.class);
  dispatcher.register(ContainerAllocatorEventType.class, containerAllocator);
  doNothing().when(containerAllocator).handle(Matchers.any(ContainerAllocatorEvent.class));
  containerLauncher = mock(ContainerLauncher.class);
  dispatcher.register(ContainerLauncherEventType.class, containerLauncher);
  doNothing().when(containerLauncher).handle(Matchers.any(ContainerLauncherEvent.class));
  workerEventDispatcher = mock(WorkerEventDispatcher.class);
  dispatcher.register(WorkerEventType.class, workerEventDispatcher);
  doNothing().when(workerEventDispatcher).handle(Matchers.any(WorkerEvent.class));
  jobEventDispatcher = new JobEventDispatcher(appContext);
  dispatcher.register(JobEventType.class, jobEventDispatcher);
  when(appContext.getEventHandler()).thenReturn(dispatcher.getEventHandler());
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入方法依赖的package包/类
protected void initDispatcher(Configuration conf) {
  // create async handler
  dispatcher = new AsyncDispatcher();
  AsyncDispatcher asyncDispatcher = (AsyncDispatcher) dispatcher;
  asyncDispatcher.init(conf);
  asyncDispatcher.setDrainEventsOnStop();
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入方法依赖的package包/类
@Override
protected void serviceInit(Configuration conf) throws Exception{
  // create async handler
  dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.register(RMStateStoreEventType.class, 
                      new ForwardingEventHandler());
  dispatcher.setDrainEventsOnStop();
  initInternal(conf);
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入方法依赖的package包/类
@Test
public void testJobNoTasks() {
  Configuration conf = new Configuration();
  conf.setInt(MRJobConfig.NUM_REDUCES, 0);
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  conf.set(MRJobConfig.WORKFLOW_ID, "testId");
  conf.set(MRJobConfig.WORKFLOW_NAME, "testName");
  conf.set(MRJobConfig.WORKFLOW_NODE_NAME, "testNodeName");
  conf.set(MRJobConfig.WORKFLOW_ADJACENCY_PREFIX_STRING + "key1", "value1");
  conf.set(MRJobConfig.WORKFLOW_ADJACENCY_PREFIX_STRING + "key2", "value2");
  conf.set(MRJobConfig.WORKFLOW_TAGS, "tag1,tag2");
  
 
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  OutputCommitter committer = mock(OutputCommitter.class);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobSubmittedEventHandler jseHandler = new JobSubmittedEventHandler("testId",
      "testName", "testNodeName", "\"key2\"=\"value2\" \"key1\"=\"value1\" ",
      "tag1,tag2");
  dispatcher.register(EventType.class, jseHandler);
  JobImpl job = createStubbedJob(conf, dispatcher, 0, null);
  job.handle(new JobEvent(job.getID(), JobEventType.JOB_INIT));
  assertJobState(job, JobStateInternal.INITED);
  job.handle(new JobStartEvent(job.getID()));
  assertJobState(job, JobStateInternal.SUCCEEDED);
  dispatcher.stop();
  commitHandler.stop();
  try {
    Assert.assertTrue(jseHandler.getAssertValue());
  } catch (InterruptedException e) {
    Assert.fail("Workflow related attributes are not tested properly");
  }
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入方法依赖的package包/类
@Test(timeout=20000)
public void testCheckJobCompleteSuccess() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  OutputCommitter committer = new TestingOutputCommitter(syncBarrier, true);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, null);
  completeJobTasks(job);
  assertJobState(job, JobStateInternal.COMMITTING);

  // let the committer complete and verify the job succeeds
  syncBarrier.await();
  assertJobState(job, JobStateInternal.SUCCEEDED);
  
  job.handle(new JobEvent(job.getID(),
      JobEventType.JOB_TASK_ATTEMPT_COMPLETED));
  assertJobState(job, JobStateInternal.SUCCEEDED);

  job.handle(new JobEvent(job.getID(), 
      JobEventType.JOB_MAP_TASK_RESCHEDULED));
  assertJobState(job, JobStateInternal.SUCCEEDED);
  
  dispatcher.stop();
  commitHandler.stop();
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入方法依赖的package包/类
@Test(timeout=20000)
public void testRebootedDuringCommit() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  conf.setInt(MRJobConfig.MR_AM_MAX_ATTEMPTS, 2);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  CyclicBarrier syncBarrier = new CyclicBarrier(2);
  OutputCommitter committer = new WaitingOutputCommitter(syncBarrier, true);
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  AppContext mockContext = mock(AppContext.class);
  when(mockContext.isLastAMRetry()).thenReturn(true);
  when(mockContext.hasSuccessfullyUnregistered()).thenReturn(false);
  JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, mockContext);
  completeJobTasks(job);
  assertJobState(job, JobStateInternal.COMMITTING);

  syncBarrier.await();
  job.handle(new JobEvent(job.getID(), JobEventType.JOB_AM_REBOOT));
  assertJobState(job, JobStateInternal.REBOOT);
  // return the external state as ERROR since this is last retry.
  Assert.assertEquals(JobState.RUNNING, job.getState());
  when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true);
  Assert.assertEquals(JobState.ERROR, job.getState());

  dispatcher.stop();
  commitHandler.stop();
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入方法依赖的package包/类
@Test(timeout=20000)
public void testKilledDuringSetup() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  OutputCommitter committer = new StubbedOutputCommitter() {
    @Override
    public synchronized void setupJob(JobContext jobContext)
        throws IOException {
      while (!Thread.interrupted()) {
        try {
          wait();
        } catch (InterruptedException e) {
        }
      }
    }
  };
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createStubbedJob(conf, dispatcher, 2, null);
  JobId jobId = job.getID();
  job.handle(new JobEvent(jobId, JobEventType.JOB_INIT));
  assertJobState(job, JobStateInternal.INITED);
  job.handle(new JobStartEvent(jobId));
  assertJobState(job, JobStateInternal.SETUP);

  job.handle(new JobEvent(job.getID(), JobEventType.JOB_KILL));
  assertJobState(job, JobStateInternal.KILLED);
  dispatcher.stop();
  commitHandler.stop();
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入方法依赖的package包/类
@Test(timeout=20000)
public void testKilledDuringKillAbort() throws Exception {
  Configuration conf = new Configuration();
  conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir);
  AsyncDispatcher dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  OutputCommitter committer = new StubbedOutputCommitter() {
    @Override
    public synchronized void abortJob(JobContext jobContext, State state)
        throws IOException {
      while (!Thread.interrupted()) {
        try {
          wait();
        } catch (InterruptedException e) {
        }
      }
    }
  };
  CommitterEventHandler commitHandler =
      createCommitterEventHandler(dispatcher, committer);
  commitHandler.init(conf);
  commitHandler.start();

  JobImpl job = createStubbedJob(conf, dispatcher, 2, null);
  JobId jobId = job.getID();
  job.handle(new JobEvent(jobId, JobEventType.JOB_INIT));
  assertJobState(job, JobStateInternal.INITED);
  job.handle(new JobStartEvent(jobId));
  assertJobState(job, JobStateInternal.SETUP);

  job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
  assertJobState(job, JobStateInternal.KILL_ABORT);

  job.handle(new JobEvent(jobId, JobEventType.JOB_KILL));
  assertJobState(job, JobStateInternal.KILLED);
  dispatcher.stop();
  commitHandler.stop();
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入方法依赖的package包/类
@Before
public void setup() {
  YarnConfiguration conf = new YarnConfiguration();
  AppContext appContext = mock(AppContext.class);

  dispatcher = new AsyncDispatcher();
  dispatcher.init(conf);
  dispatcher.start();
  taskEventDispatcher = mock(TaskEventDispatcher.class);
  dispatcher.register(TaskEventType.class, taskEventDispatcher);
  WorkerEventDispatcher workerEventDispatcher = new WorkerEventDispatcher(appContext);
  dispatcher.register(WorkerEventType.class, workerEventDispatcher);
  containerAllocator = mock(ContainerAllocator.class);
  dispatcher.register(ContainerAllocatorEventType.class, containerAllocator);
  doNothing().when(containerAllocator).handle(Matchers.any(ContainerAllocatorEvent.class));
  containerLauncher = mock(ContainerLauncher.class);
  dispatcher.register(ContainerLauncherEventType.class, containerLauncher);
  doNothing().when(containerLauncher).handle(Matchers.any(ContainerLauncherEvent.class));
  when(appContext.getEventHandler()).thenReturn(dispatcher.getEventHandler());

  TaskId taskId = mock(TaskId.class);
  when(taskId.toString()).thenReturn("taskId");
  String hostname = "localhost";
  String workerCmd = "echo hello";
  String resourceName = "centos_yarn";
  String resourcePath = "centos_yarn_path";
  YacopConfig yacopConfig = TestUtils.mockYacopConfig("simple-docker","cat /proc/1/cgroup","centos_yarn",1.0,32,2,false,null,"DOCKER");
  nWorker = new NWorkerImpl(taskId, 1, appContext.getEventHandler(), hostname, workerCmd, resourceName, resourcePath,yacopConfig);
}
 

import org.apache.hadoop.yarn.event.AsyncDispatcher; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
@Test(timeout=10000)
public void testSchedulerEventDispatcherForPreemptionEvents() {
  AsyncDispatcher rmDispatcher = new AsyncDispatcher();
  CapacityScheduler sched = spy(new CapacityScheduler());
  YarnConfiguration conf = new YarnConfiguration();
  SchedulerEventDispatcher schedulerDispatcher =
      new SchedulerEventDispatcher(sched);
  rmDispatcher.register(SchedulerEventType.class, schedulerDispatcher);
  rmDispatcher.init(conf);
  rmDispatcher.start();
  schedulerDispatcher.init(conf);
  schedulerDispatcher.start();
  try {
    ApplicationAttemptId appAttemptId = mock(ApplicationAttemptId.class);
    RMContainer container = mock(RMContainer.class);
    ContainerPreemptEvent event1 = new ContainerPreemptEvent(
        appAttemptId, container, SchedulerEventType.DROP_RESERVATION);
    rmDispatcher.getEventHandler().handle(event1);
    ContainerPreemptEvent event2 = new ContainerPreemptEvent(
         appAttemptId, container, SchedulerEventType.KILL_CONTAINER);
    rmDispatcher.getEventHandler().handle(event2);
    ContainerPreemptEvent event3 = new ContainerPreemptEvent(
        appAttemptId, container, SchedulerEventType.PREEMPT_CONTAINER);
    rmDispatcher.getEventHandler().handle(event3);
    // Wait for events to be processed by scheduler dispatcher.
    Thread.sleep(1000);
    verify(sched, times(3)).handle(any(SchedulerEvent.class));
    verify(sched).dropContainerReservation(container);
    verify(sched).preemptContainer(appAttemptId, container);
    verify(sched).killContainer(container);
  } catch (InterruptedException e) {
    Assert.fail();
  } finally {
    schedulerDispatcher.stop();
    rmDispatcher.stop();
  }
}