Java GetApplicationsRequest.newInstance方法代码示例

import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; //导入方法依赖的package包/类
protected List<ApplicationReport> fetchAppReports() throws YarnException,
    IOException {
  List<ApplicationReport> ret;
  EnumSet<YarnApplicationState> states =
      EnumSet.of(YarnApplicationState.ACCEPTED, YarnApplicationState.RUNNING);
  GetApplicationsRequest req =
      GetApplicationsRequest.newInstance(types, states);
  req.setQueues(queues);
  req.setUsers(users);
  ret = applicationReportsCache.getIfPresent(req);
  if (ret != null) {
    return ret;
  }
  ret = client.getApplications(queues, users, types, states);
  applicationReportsCache.put(req, ret);
  return ret;
}
 

import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; //导入方法依赖的package包/类
@Test
public void testApplications() throws IOException, YarnException {
  ApplicationId appId = null;
  appId = ApplicationId.newInstance(0, 1);
  ApplicationId appId1 = ApplicationId.newInstance(0, 2);
  GetApplicationsRequest request = GetApplicationsRequest.newInstance();
  GetApplicationsResponse response =
      clientService.getClientHandler().getApplications(request);
  List<ApplicationReport> appReport = response.getApplicationList();
  Assert.assertNotNull(appReport);
  Collections.sort(appReport, new Comparator<ApplicationReport>() {
    @Override
    public int compare(ApplicationReport o1, ApplicationReport o2) {
      return o1.getApplicationId().compareTo(o2.getApplicationId());
    }
  });
  Assert.assertEquals(appId, appReport.get(0).getApplicationId());
  Assert.assertEquals(appId1, appReport.get(1).getApplicationId());
}
 

import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; //导入方法依赖的package包/类
@Test
public void testLocalMode() throws IOException, YarnException, InterpreterException {
  InterpreterSetting sparkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("spark");
  sparkInterpreterSetting.setProperty("master", "local[*]");
  sparkInterpreterSetting.setProperty("SPARK_HOME", System.getenv("SPARK_HOME"));
  sparkInterpreterSetting.setProperty("ZEPPELIN_CONF_DIR", zeppelin.getZeppelinConfDir().getAbsolutePath());
  sparkInterpreterSetting.setProperty("zeppelin.spark.useHiveContext", "false");
  sparkInterpreterSetting.setProperty("zeppelin.pyspark.useIPython", "false");

  testInterpreterBasics();

  // no yarn application launched
  GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
  GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
  assertEquals(0, response.getApplicationList().size());

  interpreterSettingManager.close();
}
 

import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; //导入方法依赖的package包/类
@Test
public void testYarnClientMode() throws IOException, YarnException, InterruptedException, InterpreterException {
  InterpreterSetting sparkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("spark");
  sparkInterpreterSetting.setProperty("master", "yarn-client");
  sparkInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath());
  sparkInterpreterSetting.setProperty("SPARK_HOME", System.getenv("SPARK_HOME"));
  sparkInterpreterSetting.setProperty("ZEPPELIN_CONF_DIR", zeppelin.getZeppelinConfDir().getAbsolutePath());
  sparkInterpreterSetting.setProperty("zeppelin.spark.useHiveContext", "false");
  sparkInterpreterSetting.setProperty("zeppelin.pyspark.useIPython", "false");
  sparkInterpreterSetting.setProperty("PYSPARK_PYTHON", getPythonExec());
  sparkInterpreterSetting.setProperty("spark.driver.memory", "512m");

  testInterpreterBasics();

  // 1 yarn application launched
  GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
  GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
  assertEquals(1, response.getApplicationList().size());

  interpreterSettingManager.close();
}
 

import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; //导入方法依赖的package包/类
@Test
public void testYarnClusterMode() throws IOException, YarnException, InterruptedException, InterpreterException {
  InterpreterSetting sparkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("spark");
  sparkInterpreterSetting.setProperty("master", "yarn-cluster");
  sparkInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath());
  sparkInterpreterSetting.setProperty("SPARK_HOME", System.getenv("SPARK_HOME"));
  sparkInterpreterSetting.setProperty("ZEPPELIN_CONF_DIR", zeppelin.getZeppelinConfDir().getAbsolutePath());
  sparkInterpreterSetting.setProperty("zeppelin.spark.useHiveContext", "false");
  sparkInterpreterSetting.setProperty("zeppelin.pyspark.useIPython", "false");
  sparkInterpreterSetting.setProperty("spark.pyspark.python", getPythonExec());
  sparkInterpreterSetting.setProperty("spark.driver.memory", "512m");

  testInterpreterBasics();

  // 1 yarn application launched
  GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
  GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
  assertEquals(1, response.getApplicationList().size());

  interpreterSettingManager.close();
}
 

import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; //导入方法依赖的package包/类
@Test
public void testApplications() throws IOException, YarnException {
  ApplicationId appId = null;
  appId = ApplicationId.newInstance(0, 1);
  writeApplicationStartData(appId);
  writeApplicationFinishData(appId);
  ApplicationId appId1 = ApplicationId.newInstance(0, 2);
  writeApplicationStartData(appId1);
  writeApplicationFinishData(appId1);
  GetApplicationsRequest request = GetApplicationsRequest.newInstance();
  GetApplicationsResponse response =
      historyServer.getClientService().getClientHandler()
        .getApplications(request);
  List<ApplicationReport> appReport = response.getApplicationList();
  Assert.assertNotNull(appReport);
  Assert.assertEquals(appId, appReport.get(0).getApplicationId());
  Assert.assertEquals(appId1, appReport.get(1).getApplicationId());
}
 

import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; //导入方法依赖的package包/类
@Test
public void testApplications() throws IOException, YarnException {
  ApplicationId appId = null;
  appId = ApplicationId.newInstance(0, 1);
  ApplicationId appId1 = ApplicationId.newInstance(0, 2);
  GetApplicationsRequest request = GetApplicationsRequest.newInstance();
  GetApplicationsResponse response =
      clientService.getApplications(request);
  List<ApplicationReport> appReport = response.getApplicationList();
  Assert.assertNotNull(appReport);
  Assert.assertEquals(appId, appReport.get(0).getApplicationId());
  Assert.assertEquals(appId1, appReport.get(1).getApplicationId());
}
 

import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; //导入方法依赖的package包/类
@Override
public List<ApplicationReport> getApplications() throws YarnException,
    IOException {
  GetApplicationsRequest request = GetApplicationsRequest.newInstance(null,
      null);
  GetApplicationsResponse response = ahsClient.getApplications(request);
  return response.getApplicationList();
}
 

import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; //导入方法依赖的package包/类
@Override
public List<ApplicationReport> getApplications(Set<String> applicationTypes,
    EnumSet<YarnApplicationState> applicationStates) throws YarnException,
    IOException {
  GetApplicationsRequest request =
      GetApplicationsRequest.newInstance(applicationTypes, applicationStates);
  GetApplicationsResponse response = rmClient.getApplications(request);
  return response.getApplicationList();
}
 

import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; //导入方法依赖的package包/类
@Test
public void testApplications() throws IOException, YarnException {
  ApplicationId appId = null;
  appId = ApplicationId.newInstance(0, 1);
  ApplicationId appId1 = ApplicationId.newInstance(0, 2);
  GetApplicationsRequest request = GetApplicationsRequest.newInstance();
  GetApplicationsResponse response =
      clientService.getApplications(request);
  List<ApplicationReport> appReport = response.getApplicationList();
  Assert.assertNotNull(appReport);
  Assert.assertEquals(appId, appReport.get(1).getApplicationId());
  Assert.assertEquals(appId1, appReport.get(0).getApplicationId());

  // Create a historyManager, and set the max_apps can be loaded
  // as 1.
  Configuration conf = new YarnConfiguration();
  conf.setLong(YarnConfiguration.APPLICATION_HISTORY_MAX_APPS, 1);
  ApplicationHistoryManagerOnTimelineStore historyManager2 =
      new ApplicationHistoryManagerOnTimelineStore(dataManager,
        new ApplicationACLsManager(conf));
  historyManager2.init(conf);
  historyManager2.start();
  @SuppressWarnings("resource")
  ApplicationHistoryClientService clientService2 =
      new ApplicationHistoryClientService(historyManager2);
  response = clientService2.getApplications(request);
  appReport = response.getApplicationList();
  Assert.assertNotNull(appReport);
  Assert.assertTrue(appReport.size() == 1);
  // Expected to get the appReport for application with appId1
  Assert.assertEquals(appId1, appReport.get(0).getApplicationId());
}
 

import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; //导入方法依赖的package包/类
@Override
public List<ApplicationReport> getApplications(
    Set<String> applicationTypes) throws YarnException, IOException {
  GetApplicationsRequest request =
      applicationTypes == null ? GetApplicationsRequest.newInstance()
          : GetApplicationsRequest.newInstance(applicationTypes);
  GetApplicationsResponse response = rmClient.getApplications(request);
  return response.getApplicationList();
}
 

import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; //导入方法依赖的package包/类
@Override
public List<ApplicationReport> getApplications(
    Set<String> applicationTypes) throws YarnException, IOException {
  GetApplicationsRequest request =
      applicationTypes == null ? GetApplicationsRequest.newInstance()
          : GetApplicationsRequest.newInstance(applicationTypes);
  when(rmClient.getApplications(request))
      .thenReturn(
          getApplicationReports(reports,
              request));
  GetApplicationsResponse response = rmClient.getApplications(request);
  return response.getApplicationList();
}
 

import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; //导入方法依赖的package包/类
@Test
public void testGetApplicationsRequest(){
  GetApplicationsRequest request = GetApplicationsRequest.newInstance();
  
  EnumSet<YarnApplicationState> appStates = 
    EnumSet.of(YarnApplicationState.ACCEPTED);
  request.setApplicationStates(appStates);
  
  Set<String> tags = new HashSet<String>();
  tags.add("tag1");
  request.setApplicationTags(tags);
  
  Set<String> types = new HashSet<String>();
  types.add("type1");
  request.setApplicationTypes(types);
  
  long startBegin = System.currentTimeMillis();
  long startEnd = System.currentTimeMillis() + 1;
  request.setStartRange(startBegin, startEnd);
  long finishBegin = System.currentTimeMillis() + 2;
  long finishEnd = System.currentTimeMillis() + 3;
  request.setFinishRange(finishBegin, finishEnd);
  
  long limit = 100L;
  request.setLimit(limit);
  
  Set<String> queues = new HashSet<String>();
  queues.add("queue1");
  request.setQueues(queues);
  
  
  Set<String> users = new HashSet<String>();
  users.add("user1");
  request.setUsers(users);
  
  ApplicationsRequestScope scope = ApplicationsRequestScope.ALL;
  request.setScope(scope);
  
  GetApplicationsRequest requestFromProto = new GetApplicationsRequestPBImpl(
      ((GetApplicationsRequestPBImpl)request).getProto());
  
  // verify the whole record equals with original record
  Assert.assertEquals(requestFromProto, request);

  // verify all properties are the same as original request
  Assert.assertEquals(
      "ApplicationStates from proto is not the same with original request",
      requestFromProto.getApplicationStates(), appStates);
  
  Assert.assertEquals(
      "ApplicationTags from proto is not the same with original request",
      requestFromProto.getApplicationTags(), tags);
  
  Assert.assertEquals(
      "ApplicationTypes from proto is not the same with original request",
      requestFromProto.getApplicationTypes(), types);
  
  Assert.assertEquals(
      "StartRange from proto is not the same with original request",
      requestFromProto.getStartRange(), new LongRange(startBegin, startEnd));
  
  Assert.assertEquals(
      "FinishRange from proto is not the same with original request",
      requestFromProto.getFinishRange(), new LongRange(finishBegin, finishEnd));
  
  Assert.assertEquals(
      "Limit from proto is not the same with original request",
      requestFromProto.getLimit(), limit);
  
  Assert.assertEquals(
      "Queues from proto is not the same with original request",
      requestFromProto.getQueues(), queues);
  
  Assert.assertEquals(
      "Users from proto is not the same with original request",
      requestFromProto.getUsers(), users);
}
 

import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; //导入方法依赖的package包/类
@Test (timeout = 80000)
public void testInvalidateAMHostPortWhenAMFailedOrKilled() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
  MockRM rm1 = new MockRM(conf);
  rm1.start();

  // a succeeded app
  RMApp app1 = rm1.submitApp(200);
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1);

  // a failed app
  RMApp app2 = rm1.submitApp(200);
  MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
  nm1.nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  am2.waitForState(RMAppAttemptState.FAILED);
  rm1.waitForState(app2.getApplicationId(), RMAppState.FAILED);

  // a killed app
  RMApp app3 = rm1.submitApp(200);
  MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1);
  rm1.killApp(app3.getApplicationId());
  rm1.waitForState(app3.getApplicationId(), RMAppState.KILLED);
  rm1.waitForState(am3.getApplicationAttemptId(), RMAppAttemptState.KILLED);

  GetApplicationsRequest request1 =
      GetApplicationsRequest.newInstance(EnumSet.of(
        YarnApplicationState.FINISHED, YarnApplicationState.KILLED,
        YarnApplicationState.FAILED));
  GetApplicationsResponse response1 =
      rm1.getClientRMService().getApplications(request1);
  List<ApplicationReport> appList1 = response1.getApplicationList();

  Assert.assertEquals(3, appList1.size());
  for (ApplicationReport report : appList1) {
    // killed/failed apps host and rpc port are invalidated.
    if (report.getApplicationId().equals(app2.getApplicationId())
        || report.getApplicationId().equals(app3.getApplicationId())) {
      Assert.assertEquals("N/A", report.getHost());
      Assert.assertEquals(-1, report.getRpcPort());
    }
    // succeeded app's host and rpc port is not invalidated
    if (report.getApplicationId().equals(app1.getApplicationId())) {
      Assert.assertFalse(report.getHost().equals("N/A"));
      Assert.assertTrue(report.getRpcPort() != -1);
    }
  }
}
 

import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; //导入方法依赖的package包/类
@Test
public void testForceKillApplication() throws Exception {
  YarnConfiguration conf = new YarnConfiguration();
  MockRM rm = new MockRM();
  rm.init(conf);
  rm.start();

  ClientRMService rmService = rm.getClientRMService();
  GetApplicationsRequest getRequest = GetApplicationsRequest.newInstance(
      EnumSet.of(YarnApplicationState.KILLED));

  RMApp app1 = rm.submitApp(1024);
  RMApp app2 = rm.submitApp(1024, true);

  assertEquals("Incorrect number of apps in the RM", 0,
      rmService.getApplications(getRequest).getApplicationList().size());

  KillApplicationRequest killRequest1 =
      KillApplicationRequest.newInstance(app1.getApplicationId());
  KillApplicationRequest killRequest2 =
      KillApplicationRequest.newInstance(app2.getApplicationId());

  int killAttemptCount = 0;
  for (int i = 0; i < 100; i++) {
    KillApplicationResponse killResponse1 =
        rmService.forceKillApplication(killRequest1);
    killAttemptCount++;
    if (killResponse1.getIsKillCompleted()) {
      break;
    }
    Thread.sleep(10);
  }
  assertTrue("Kill attempt count should be greater than 1 for managed AMs",
      killAttemptCount > 1);
  assertEquals("Incorrect number of apps in the RM", 1,
      rmService.getApplications(getRequest).getApplicationList().size());

  KillApplicationResponse killResponse2 =
      rmService.forceKillApplication(killRequest2);
  assertTrue("Killing UnmanagedAM should falsely acknowledge true",
      killResponse2.getIsKillCompleted());
  for (int i = 0; i < 100; i++) {
    if (2 ==
        rmService.getApplications(getRequest).getApplicationList().size()) {
      break;
    }
    Thread.sleep(10);
  }
  assertEquals("Incorrect number of apps in the RM", 2,
      rmService.getApplications(getRequest).getApplicationList().size());
}