Java YarnRPC.getProxy方法代码示例

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
@Before
public void startUp() {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.SCM_STORE_CLASS,
      InMemorySCMStore.class.getName());
  conf.set(YarnConfiguration.SHARED_CACHE_ROOT, testDir.getPath());
  AppChecker appChecker = spy(new DummyAppChecker());
  store = new InMemorySCMStore(appChecker);
  store.init(conf);
  store.start();

  service = new SharedCacheUploaderService(store);
  service.init(conf);
  service.start();

  YarnRPC rpc = YarnRPC.create(new Configuration());

  InetSocketAddress scmAddress =
      conf.getSocketAddr(YarnConfiguration.SCM_UPLOADER_SERVER_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_PORT);

  proxy =
      (SCMUploaderProtocol) rpc.getProxy(
          SCMUploaderProtocol.class, scmAddress, conf);
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
@Before
public void startUp() {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.SCM_STORE_CLASS,
      InMemorySCMStore.class.getName());
  conf.set(YarnConfiguration.SHARED_CACHE_ROOT, testDir.getPath());
  AppChecker appChecker = spy(new DummyAppChecker());
  store = new InMemorySCMStore(appChecker);
  store.init(conf);
  store.start();

  service = new ClientProtocolService(store);
  service.init(conf);
  service.start();

  YarnRPC rpc = YarnRPC.create(new Configuration());

  InetSocketAddress scmAddress =
      conf.getSocketAddr(YarnConfiguration.SCM_CLIENT_SERVER_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_CLIENT_SERVER_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_CLIENT_SERVER_PORT);

  clientSCMProxy =
      (ClientSCMProtocol) rpc.getProxy(ClientSCMProtocol.class, scmAddress,
          conf);
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
@Test
public void testLocalizerRPC() throws Exception {
  InetSocketAddress locAddr = new InetSocketAddress("0.0.0.0", 8040);
  LocalizerService server = new LocalizerService(locAddr);
  try {
    server.start();
    Configuration conf = new Configuration();
    YarnRPC rpc = YarnRPC.create(conf);
    LocalizationProtocol client = (LocalizationProtocol)
      rpc.getProxy(LocalizationProtocol.class, locAddr, conf);
    LocalizerStatus status =
      recordFactory.newRecordInstance(LocalizerStatus.class);
    status.setLocalizerId("localizer0");
    LocalizerHeartbeatResponse response = client.heartbeat(status);
    assertEquals(dieHBResponse(), response);
  } finally {
    server.stop();
  }
  assertTrue(true);
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
@Before
public void startUp() {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.SCM_STORE_CLASS,
      InMemorySCMStore.class.getName());

  cleaner = mock(CleanerService.class);

  service = spy(new SCMAdminProtocolService(cleaner));
  service.init(conf);
  service.start();

  YarnRPC rpc = YarnRPC.create(new Configuration());

  InetSocketAddress scmAddress =
      conf.getSocketAddr(YarnConfiguration.SCM_ADMIN_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_ADMIN_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_ADMIN_PORT);

  SCMAdminProxy =
      (SCMAdminProtocol) rpc.getProxy(SCMAdminProtocol.class, scmAddress,
          conf);

  mockAdmin = mock(SCMAdminProtocol.class);
  adminCLI = new SCMAdmin(new Configuration()) {
    @Override
    protected SCMAdminProtocol createSCMAdminProtocol() throws IOException {
      return mockAdmin;
    }
  };
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
private SCMUploaderProtocol createSCMClient(Configuration conf) {
  YarnRPC rpc = YarnRPC.create(conf);
  InetSocketAddress scmAddress =
      conf.getSocketAddr(YarnConfiguration.SCM_UPLOADER_SERVER_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_PORT);
  return (SCMUploaderProtocol)rpc.getProxy(
      SCMUploaderProtocol.class, scmAddress, conf);
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
protected SCMAdminProtocol createSCMAdminProtocol() throws IOException {
  // Get the current configuration
  final YarnConfiguration conf = new YarnConfiguration(getConf());

  // Create the admin client
  final InetSocketAddress addr = conf.getSocketAddr(
      YarnConfiguration.SCM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_SCM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_SCM_ADMIN_PORT);
  final YarnRPC rpc = YarnRPC.create(conf);
  SCMAdminProtocol scmAdminProtocol =
      (SCMAdminProtocol) rpc.getProxy(SCMAdminProtocol.class, addr, conf);
  return scmAdminProtocol;
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
@Override
protected GetUserMappingsProtocol getUgmProtocol() throws IOException {
  Configuration conf = getConf();
  
  final InetSocketAddress addr = conf.getSocketAddr(
      YarnConfiguration.RM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_RM_ADMIN_PORT);
  final YarnRPC rpc = YarnRPC.create(conf);
  
  ResourceManagerAdministrationProtocol adminProtocol = (ResourceManagerAdministrationProtocol) rpc.getProxy(
      ResourceManagerAdministrationProtocol.class, addr, getConf());

  return adminProtocol;
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
MRClientProtocol instantiateAMProxy(final InetSocketAddress serviceAddr)
    throws IOException {
  LOG.trace("Connecting to ApplicationMaster at: " + serviceAddr);
  YarnRPC rpc = YarnRPC.create(conf);
  MRClientProtocol proxy = 
       (MRClientProtocol) rpc.getProxy(MRClientProtocol.class,
          serviceAddr, conf);
  usingAMProxy.set(true);
  LOG.trace("Connected to ApplicationMaster at: " + serviceAddr);
  return proxy;
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
private HSClientProtocol instantiateHistoryProxy() {
  final String serviceAddr =
      mrCluster.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS);
  final YarnRPC rpc = YarnRPC.create(conf);
  HSClientProtocol historyClient =
      (HSClientProtocol) rpc.getProxy(HSClientProtocol.class,
          NetUtils.createSocketAddr(serviceAddr), mrCluster.getConfig());
  return historyClient;
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
private void test(String rpcClass) throws Exception {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
  YarnRPC rpc = YarnRPC.create(conf);
  String bindAddr = "localhost:0";
  InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
  Server server = rpc.getServer(ContainerManagementProtocol.class, 
          new DummyContainerManager(), addr, conf, null, 1);
  server.start();
  RPC.setProtocolEngine(conf, ContainerManagementProtocolPB.class, ProtobufRpcEngine.class);
  ContainerManagementProtocol proxy = (ContainerManagementProtocol) 
      rpc.getProxy(ContainerManagementProtocol.class, 
          NetUtils.getConnectAddress(server), conf);
  ContainerLaunchContext containerLaunchContext = 
      recordFactory.newRecordInstance(ContainerLaunchContext.class);

  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId applicationAttemptId =
      ApplicationAttemptId.newInstance(applicationId, 0);
  ContainerId containerId =
      ContainerId.newContainerId(applicationAttemptId, 100);
  NodeId nodeId = NodeId.newInstance("localhost", 1234);
  Resource resource = Resource.newInstance(1234, 2);
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier(containerId, "localhost", "user",
        resource, System.currentTimeMillis() + 10000, 42, 42,
        Priority.newInstance(0), 0);
  Token containerToken = newContainerToken(nodeId, "password".getBytes(),
        containerTokenIdentifier);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(containerLaunchContext,
        containerToken);
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  proxy.startContainers(allRequests);

  List<ContainerId> containerIds = new ArrayList<ContainerId>();
  containerIds.add(containerId);
  GetContainerStatusesRequest gcsRequest =
      GetContainerStatusesRequest.newInstance(containerIds);
  GetContainerStatusesResponse response =
      proxy.getContainerStatuses(gcsRequest);
  List<ContainerStatus> statuses = response.getContainerStatuses();

  //test remote exception
  boolean exception = false;
  try {
    StopContainersRequest stopRequest =
        recordFactory.newRecordInstance(StopContainersRequest.class);
    stopRequest.setContainerIds(containerIds);
    proxy.stopContainers(stopRequest);
    } catch (YarnException e) {
    exception = true;
    Assert.assertTrue(e.getMessage().contains(EXCEPTION_MSG));
    Assert.assertTrue(e.getMessage().contains(EXCEPTION_CAUSE));
    System.out.println("Test Exception is " + e.getMessage());
  } catch (Exception ex) {
    ex.printStackTrace();
  }
  Assert.assertTrue(exception);
  
  server.stop();
  Assert.assertNotNull(statuses.get(0));
  Assert.assertEquals(ContainerState.RUNNING, statuses.get(0).getState());
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
private void testRPCTimeout(String rpcClass) throws Exception {
  Configuration conf = new Configuration();
  // set timeout low for the test
  conf.setInt("yarn.rpc.nm-command-timeout", 3000);

  conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
  YarnRPC rpc = YarnRPC.create(conf);
  String bindAddr = "localhost:0";
  InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
  Server server = rpc.getServer(ContainerManagementProtocol.class,
      new DummyContainerManager(), addr, conf, null, 1);
  server.start();
  try {

    ContainerManagementProtocol proxy = (ContainerManagementProtocol) rpc.getProxy(
        ContainerManagementProtocol.class,
        server.getListenerAddress(), conf);
    ContainerLaunchContext containerLaunchContext = recordFactory
        .newRecordInstance(ContainerLaunchContext.class);

    ApplicationId applicationId = ApplicationId.newInstance(0, 0);
    ApplicationAttemptId applicationAttemptId =
        ApplicationAttemptId.newInstance(applicationId, 0);
    ContainerId containerId =
        ContainerId.newContainerId(applicationAttemptId, 100);
    NodeId nodeId = NodeId.newInstance("localhost", 1234);
    Resource resource = Resource.newInstance(1234, 2, 3);
    ContainerTokenIdentifier containerTokenIdentifier =
        new ContainerTokenIdentifier(containerId, "localhost", "user",
          resource, System.currentTimeMillis() + 10000, 42, 42,
          Priority.newInstance(0), 0);
    Token containerToken =
        TestRPC.newContainerToken(nodeId, "password".getBytes(),
          containerTokenIdentifier);

    StartContainerRequest scRequest =
        StartContainerRequest.newInstance(containerLaunchContext,
          containerToken);
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests =
        StartContainersRequest.newInstance(list);
    try {
      proxy.startContainers(allRequests);
    } catch (Exception e) {
      LOG.info(StringUtils.stringifyException(e));
      Assert.assertEquals("Error, exception is not: "
          + SocketTimeoutException.class.getName(),
          SocketTimeoutException.class.getName(), e.getClass().getName());
      return;
    }
  } finally {
    server.stop();
  }

  Assert.fail("timeout exception should have occurred!");
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
LocalizationProtocol getProxy(final InetSocketAddress nmAddr) {
  YarnRPC rpc = YarnRPC.create(conf);
  return (LocalizationProtocol)
    rpc.getProxy(LocalizationProtocol.class, nmAddr, conf);
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
@Test
public void testGetNodeLabels() throws Exception {
  MockRM rm = new MockRM() {
    protected ClientRMService createClientRMService() {
      return new ClientRMService(this.rmContext, scheduler,
          this.rmAppManager, this.applicationACLsManager,
          this.queueACLsManager, this.getRMContext()
              .getRMDelegationTokenSecretManager());
    };
  };
  rm.start();
  RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager();
  labelsMgr.addToCluserNodeLabels(ImmutableSet.of("x", "y"));

  Map<NodeId, Set<String>> map = new HashMap<NodeId, Set<String>>();
  map.put(NodeId.newInstance("host1", 0), ImmutableSet.of("x"));
  map.put(NodeId.newInstance("host2", 0), ImmutableSet.of("y"));
  labelsMgr.replaceLabelsOnNode(map);

  // Create a client.
  Configuration conf = new Configuration();
  YarnRPC rpc = YarnRPC.create(conf);
  InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
  LOG.info("Connecting to ResourceManager at " + rmAddress);
  ApplicationClientProtocol client =
      (ApplicationClientProtocol) rpc.getProxy(
          ApplicationClientProtocol.class, rmAddress, conf);

  // Get node labels collection
  GetClusterNodeLabelsResponse response =
      client.getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance());
  Assert.assertTrue(response.getNodeLabels().containsAll(
      Arrays.asList("x", "y")));

  // Get node labels mapping
  GetNodesToLabelsResponse response1 =
      client.getNodeToLabels(GetNodesToLabelsRequest.newInstance());
  Map<NodeId, Set<String>> nodeToLabels = response1.getNodeToLabels();
  Assert.assertTrue(nodeToLabels.keySet().containsAll(
      Arrays.asList(NodeId.newInstance("host1", 0),
          NodeId.newInstance("host2", 0))));
  Assert.assertTrue(nodeToLabels.get(NodeId.newInstance("host1", 0))
      .containsAll(Arrays.asList("x")));
  Assert.assertTrue(nodeToLabels.get(NodeId.newInstance("host2", 0))
      .containsAll(Arrays.asList("y")));
  
  rpc.stopProxy(client, conf);
  rm.close();
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
@Test
public void testGetLabelsToNodes() throws Exception {
  MockRM rm = new MockRM() {
    protected ClientRMService createClientRMService() {
      return new ClientRMService(this.rmContext, scheduler,
          this.rmAppManager, this.applicationACLsManager,
          this.queueACLsManager, this.getRMContext()
              .getRMDelegationTokenSecretManager());
    };
  };
  rm.start();
  RMNodeLabelsManager labelsMgr = rm.getRMContext().getNodeLabelManager();
  labelsMgr.addToCluserNodeLabels(ImmutableSet.of("x", "y", "z"));

  Map<NodeId, Set<String>> map = new HashMap<NodeId, Set<String>>();
  map.put(NodeId.newInstance("host1", 0), ImmutableSet.of("x"));
  map.put(NodeId.newInstance("host1", 1), ImmutableSet.of("z"));
  map.put(NodeId.newInstance("host2", 0), ImmutableSet.of("y"));
  map.put(NodeId.newInstance("host3", 0), ImmutableSet.of("y"));
  map.put(NodeId.newInstance("host3", 1), ImmutableSet.of("z"));
  labelsMgr.replaceLabelsOnNode(map);

  // Create a client.
  Configuration conf = new Configuration();
  YarnRPC rpc = YarnRPC.create(conf);
  InetSocketAddress rmAddress = rm.getClientRMService().getBindAddress();
  LOG.info("Connecting to ResourceManager at " + rmAddress);
  ApplicationClientProtocol client =
      (ApplicationClientProtocol) rpc.getProxy(
          ApplicationClientProtocol.class, rmAddress, conf);

  // Get node labels collection
  GetClusterNodeLabelsResponse response =
      client.getClusterNodeLabels(GetClusterNodeLabelsRequest.newInstance());
  Assert.assertTrue(response.getNodeLabels().containsAll(
      Arrays.asList("x", "y", "z")));

  // Get labels to nodes mapping
  GetLabelsToNodesResponse response1 =
      client.getLabelsToNodes(GetLabelsToNodesRequest.newInstance());
  Map<String, Set<NodeId>> labelsToNodes = response1.getLabelsToNodes();
  Assert.assertTrue(
      labelsToNodes.keySet().containsAll(Arrays.asList("x", "y", "z")));
  Assert.assertTrue(
      labelsToNodes.get("x").containsAll(Arrays.asList(
      NodeId.newInstance("host1", 0))));
  Assert.assertTrue(
      labelsToNodes.get("y").containsAll(Arrays.asList(
      NodeId.newInstance("host2", 0), NodeId.newInstance("host3", 0))));
  Assert.assertTrue(
      labelsToNodes.get("z").containsAll(Arrays.asList(
      NodeId.newInstance("host1", 1), NodeId.newInstance("host3", 1))));

  // Get labels to nodes mapping for specific labels
  Set<String> setlabels =
      new HashSet<String>(Arrays.asList(new String[]{"x", "z"}));
  GetLabelsToNodesResponse response2 =
      client.getLabelsToNodes(GetLabelsToNodesRequest.newInstance(setlabels));
  labelsToNodes = response2.getLabelsToNodes();
  Assert.assertTrue(
      labelsToNodes.keySet().containsAll(Arrays.asList("x", "z")));
  Assert.assertTrue(
      labelsToNodes.get("x").containsAll(Arrays.asList(
      NodeId.newInstance("host1", 0))));
  Assert.assertTrue(
      labelsToNodes.get("z").containsAll(Arrays.asList(
      NodeId.newInstance("host1", 1), NodeId.newInstance("host3", 1))));
  Assert.assertEquals(labelsToNodes.get("y"), null);

  rpc.stopProxy(client, conf);
  rm.close();
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
@VisibleForTesting
protected ClientSCMProtocol createClientProxy() {
  YarnRPC rpc = YarnRPC.create(getConfig());
  return (ClientSCMProtocol) rpc.getProxy(ClientSCMProtocol.class,
      this.scmAddress, getConfig());
}