Java YarnRPC.create方法代码示例

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
public void start(Configuration conf) {
  YarnRPC rpc = YarnRPC.create(conf);
  //TODO : use fixed port ??
  InetSocketAddress address = NetUtils.createSocketAddr(hostAddress);
  InetAddress hostNameResolved = null;
  try {
    address.getAddress();
    hostNameResolved = InetAddress.getLocalHost();
  } catch (UnknownHostException e) {
    throw new YarnRuntimeException(e);
  }

  server =
      rpc.getServer(protocol, this, address,
          conf, null, 1);
  server.start();
  this.bindAddress = NetUtils.getConnectAddress(server);
   super.start();
   amRunning = true;
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
Server createServer() {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  if (UserGroupInformation.isSecurityEnabled()) {
    secretManager = new LocalizerTokenSecretManager();      
  }
  
  Server server = rpc.getServer(LocalizationProtocol.class, this,
      localizationServerAddress, conf, secretManager, 
      conf.getInt(YarnConfiguration.NM_LOCALIZER_CLIENT_THREAD_COUNT, 
          YarnConfiguration.DEFAULT_NM_LOCALIZER_CLIENT_THREAD_COUNT));
  
  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
      false)) {
    server.refreshServiceAcl(conf, new NMPolicyProvider());
  }
  
  return server;
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
@Test
public void testLocalizerRPC() throws Exception {
  InetSocketAddress locAddr = new InetSocketAddress("0.0.0.0", 8040);
  LocalizerService server = new LocalizerService(locAddr);
  try {
    server.start();
    Configuration conf = new Configuration();
    YarnRPC rpc = YarnRPC.create(conf);
    LocalizationProtocol client = (LocalizationProtocol)
      rpc.getProxy(LocalizationProtocol.class, locAddr, conf);
    LocalizerStatus status =
      recordFactory.newRecordInstance(LocalizerStatus.class);
    status.setLocalizerId("localizer0");
    LocalizerHeartbeatResponse response = client.heartbeat(status);
    assertEquals(dieHBResponse(), response);
  } finally {
    server.stop();
  }
  assertTrue(true);
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
@Override
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  this.server =
      rpc.getServer(SCMAdminProtocol.class, this,
          clientBindAddress,
          conf, null, // Secret manager null for now (security not supported)
          conf.getInt(YarnConfiguration.SCM_ADMIN_CLIENT_THREAD_COUNT,
              YarnConfiguration.DEFAULT_SCM_ADMIN_CLIENT_THREAD_COUNT));

  // TODO: Enable service authorization (see YARN-2774)

  this.server.start();
  clientBindAddress =
      conf.updateConnectAddr(YarnConfiguration.SCM_ADMIN_ADDRESS,
          server.getListenerAddress());

  super.serviceStart();
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
private ApplicationClientProtocol getClientRMProtocolWithDT(
    org.apache.hadoop.yarn.api.records.Token token,
    final InetSocketAddress rmAddress, String user, final Configuration conf) {
  // Maybe consider converting to Hadoop token, serialize de-serialize etc
  // before trying to renew the token.

  UserGroupInformation ugi = UserGroupInformation
      .createRemoteUser(user);
  ugi.addToken(ConverterUtils.convertFromYarn(token, rmAddress));

  final YarnRPC rpc = YarnRPC.create(conf);
  ApplicationClientProtocol clientRMWithDT = ugi
      .doAs(new PrivilegedAction<ApplicationClientProtocol>() {
        @Override
        public ApplicationClientProtocol run() {
          return (ApplicationClientProtocol) rpc.getProxy(ApplicationClientProtocol.class,
              rmAddress, conf);
        }
      });
  return clientRMWithDT;
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
protected MRClientProtocol instantiateHistoryProxy()
    throws IOException {
  final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS);
  if (StringUtils.isEmpty(serviceAddr)) {
    return null;
  }
  LOG.debug("Connecting to HistoryServer at: " + serviceAddr);
  final YarnRPC rpc = YarnRPC.create(conf);
  LOG.debug("Connected to HistoryServer at: " + serviceAddr);
  UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
  return currentUser.doAs(new PrivilegedAction<MRClientProtocol>() {
    @Override
    public MRClientProtocol run() {
      return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class,
          NetUtils.createSocketAddr(serviceAddr), conf);
    }
  });
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
@Override
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  this.metrics = SharedCacheUploaderMetrics.getInstance();

  YarnRPC rpc = YarnRPC.create(conf);
  this.server =
      rpc.getServer(SCMUploaderProtocol.class, this, bindAddress,
          conf, null, // Secret manager null for now (security not supported)
          conf.getInt(YarnConfiguration.SCM_UPLOADER_SERVER_THREAD_COUNT,
              YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_THREAD_COUNT));

  // TODO (YARN-2774): Enable service authorization

  this.server.start();
  bindAddress =
      conf.updateConnectAddr(YarnConfiguration.SCM_UPLOADER_SERVER_ADDRESS,
          server.getListenerAddress());

  super.serviceStart();
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
@Override
public void run() {
  Configuration conf = new Configuration();
  YarnRPC rpc = YarnRPC.create(conf);
  InetSocketAddress address = new InetSocketAddress(rpcAddress, rpcPort);
  Server server = rpc.getServer(
      TensorFlowCluster.class, this, address, conf, null,
      conf.getInt(YarnConfiguration.RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT,
          YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_CLIENT_THREAD_COUNT));

  server.start();
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
public ContainerManagementProtocolProxy(Configuration conf,
    NMTokenCache nmTokenCache) {
  this.conf = new Configuration(conf);
  this.nmTokenCache = nmTokenCache;

  maxConnectedNMs =
      conf.getInt(YarnConfiguration.NM_CLIENT_MAX_NM_PROXIES,
          YarnConfiguration.DEFAULT_NM_CLIENT_MAX_NM_PROXIES);
  if (maxConnectedNMs < 0) {
    throw new YarnRuntimeException(
        YarnConfiguration.NM_CLIENT_MAX_NM_PROXIES
            + " (" + maxConnectedNMs + ") can not be less than 0.");
  }
  LOG.info(YarnConfiguration.NM_CLIENT_MAX_NM_PROXIES + " : "
      + maxConnectedNMs);

  if (maxConnectedNMs > 0) {
    cmProxy =
        new LinkedHashMap<String, ContainerManagementProtocolProxyData>();
  } else {
    cmProxy = Collections.emptyMap();
    // Connections are not being cached so ensure connections close quickly
    // to avoid creating thousands of RPC client threads on large clusters.
    this.conf.setInt(
        CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
        0);
  }
  rpc = YarnRPC.create(conf);
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
@Override
protected void serviceStart() throws Exception {
  // All the clients to appsManager are supposed to be authenticated via
  // Kerberos if security is enabled, so no secretManager.
  YarnRPC rpc = YarnRPC.create(getConfig());
  Configuration clientServerConf = new Configuration(getConfig());
  this.server = rpc.getServer(ApplicationClientProtocol.class, this,
      clientBindAddress, clientServerConf, null, 1);
  this.server.start();
  super.serviceStart();
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
private HSClientProtocol instantiateHistoryProxy() {
  final String serviceAddr =
      mrCluster.getConfig().get(JHAdminConfig.MR_HISTORY_ADDRESS);
  final YarnRPC rpc = YarnRPC.create(conf);
  HSClientProtocol historyClient =
      (HSClientProtocol) rpc.getProxy(HSClientProtocol.class,
          NetUtils.createSocketAddr(serviceAddr), mrCluster.getConfig());
  return historyClient;
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
@Override
protected GetUserMappingsProtocol getUgmProtocol() throws IOException {
  Configuration conf = getConf();
  
  final InetSocketAddress addr = conf.getSocketAddr(
      YarnConfiguration.RM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
      YarnConfiguration.DEFAULT_RM_ADMIN_PORT);
  final YarnRPC rpc = YarnRPC.create(conf);
  
  ResourceManagerAdministrationProtocol adminProtocol = (ResourceManagerAdministrationProtocol) rpc.getProxy(
      ResourceManagerAdministrationProtocol.class, addr, getConf());

  return adminProtocol;
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
protected void serviceStart() throws Exception {
  Configuration conf = getConfig();
  YarnRPC rpc = YarnRPC.create(conf);
  InetSocketAddress address = new InetSocketAddress(0);

  server =
      rpc.getServer(MRClientProtocol.class, protocolHandler, address,
          conf, appContext.getClientToAMTokenSecretManager(),
          conf.getInt(MRJobConfig.MR_AM_JOB_CLIENT_THREAD_COUNT, 
              MRJobConfig.DEFAULT_MR_AM_JOB_CLIENT_THREAD_COUNT),
              MRJobConfig.MR_AM_JOB_CLIENT_PORT_RANGE);
  
  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
      false)) {
    refreshServiceAcls(conf, new MRAMPolicyProvider());
  }

  server.start();
  this.bindAddress = NetUtils.createSocketAddrForHost(appContext.getNMHostname(),
      server.getListenerAddress().getPort());
  LOG.info("Instantiated MRClientService at " + this.bindAddress);
  try {
    // Explicitly disabling SSL for map reduce task as we can't allow MR users
    // to gain access to keystore file for opening SSL listener. We can trust
    // RM/NM to issue SSL certificates but definitely not MR-AM as it is
    // running in user-land.
    webApp =
        WebApps.$for("mapreduce", AppContext.class, appContext, "ws")
          .withHttpPolicy(conf, Policy.HTTP_ONLY).start(new AMWebApp());
  } catch (Exception e) {
    LOG.error("Webapps failed to start. Ignoring for now:", e);
  }
  super.serviceStart();
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
private SCMUploaderProtocol createSCMClient(Configuration conf) {
  YarnRPC rpc = YarnRPC.create(conf);
  InetSocketAddress scmAddress =
      conf.getSocketAddr(YarnConfiguration.SCM_UPLOADER_SERVER_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_ADDRESS,
          YarnConfiguration.DEFAULT_SCM_UPLOADER_SERVER_PORT);
  return (SCMUploaderProtocol)rpc.getProxy(
      SCMUploaderProtocol.class, scmAddress, conf);
}
 

import org.apache.hadoop.yarn.ipc.YarnRPC; //导入方法依赖的package包/类
@Override
protected ContainerLauncher
    createContainerLauncher(final AppContext context) {
  return new ContainerLauncherImpl(context) {

    @Override
    public ContainerManagementProtocolProxyData getCMProxy(
        String containerMgrBindAddr, ContainerId containerId)
        throws IOException {
      InetSocketAddress addr = NetUtils.getConnectAddress(server);
      String containerManagerBindAddr =
          addr.getHostName() + ":" + addr.getPort();
      Token token =
          tokenSecretManager.createNMToken(
            containerId.getApplicationAttemptId(),
            NodeId.newInstance(addr.getHostName(), addr.getPort()), "user");
      ContainerManagementProtocolProxy cmProxy =
          new ContainerManagementProtocolProxy(conf);
      ContainerManagementProtocolProxyData proxy =
          cmProxy.new ContainerManagementProtocolProxyData(
            YarnRPC.create(conf), containerManagerBindAddr, containerId,
            token);
      return proxy;
    }
  };

}