Java MRClientProtocol类代码示例

import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; //导入依赖的package包/类
@VisibleForTesting
void addHistoryToken(Credentials ts) throws IOException, InterruptedException {
	/* check if we have a hsproxy, if not, no need */
	MRClientProtocol hsProxy = clientCache.getInitializedHSProxy();
	if (UserGroupInformation.isSecurityEnabled() && (hsProxy != null)) {
		/*
		 * note that get delegation token was called. Again this is hack for
		 * oozie to make sure we add history server delegation tokens to the
		 * credentials
		 */
		RMDelegationTokenSelector tokenSelector = new RMDelegationTokenSelector();
		Text service = resMgrDelegate.getRMDelegationTokenService();
		if (tokenSelector.selectToken(service, ts.getAllTokens()) != null) {
			Text hsService = SecurityUtil.buildTokenService(hsProxy.getConnectAddress());
			if (ts.getToken(hsService) == null) {
				ts.addToken(hsService, getDelegationTokenFromHS(hsProxy));
			}
		}
	}
}
 

import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; //导入依赖的package包/类
protected MRClientProtocol instantiateHistoryProxy()
    throws IOException {
  final String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS);
  if (StringUtils.isEmpty(serviceAddr)) {
    return null;
  }
  LOG.debug("Connecting to HistoryServer at: " + serviceAddr);
  final YarnRPC rpc = YarnRPC.create(conf);
  LOG.debug("Connected to HistoryServer at: " + serviceAddr);
  UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
  return currentUser.doAs(new PrivilegedAction<MRClientProtocol>() {
    @Override
    public MRClientProtocol run() {
      return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class,
          NetUtils.createSocketAddr(serviceAddr), conf);
    }
  });
}
 

import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; //导入依赖的package包/类
public ClientServiceDelegate(Configuration conf, ResourceMgrDelegate rm,
    JobID jobId, MRClientProtocol historyServerProxy) {
  this.conf = new Configuration(conf); // Cloning for modifying.
  // For faster redirects from AM to HS.
  this.conf.setInt(
      CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,
      this.conf.getInt(MRJobConfig.MR_CLIENT_TO_AM_IPC_MAX_RETRIES,
          MRJobConfig.DEFAULT_MR_CLIENT_TO_AM_IPC_MAX_RETRIES));
  this.conf.setInt(
      CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
      this.conf.getInt(MRJobConfig.MR_CLIENT_TO_AM_IPC_MAX_RETRIES_ON_TIMEOUTS,
          MRJobConfig.DEFAULT_MR_CLIENT_TO_AM_IPC_MAX_RETRIES_ON_TIMEOUTS));
  this.rm = rm;
  this.jobId = jobId;
  this.historyServerProxy = historyServerProxy;
  this.appId = TypeConverter.toYarn(jobId).getAppId();
  notRunningJobs = new HashMap<JobState, HashMap<String, NotRunningJob>>();
}
 

import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; //导入依赖的package包/类
@VisibleForTesting
void addHistoryToken(Credentials ts) throws IOException, InterruptedException {
  /* check if we have a hsproxy, if not, no need */
  MRClientProtocol hsProxy = clientCache.getInitializedHSProxy();
  if (UserGroupInformation.isSecurityEnabled() && (hsProxy != null)) {
    /*
     * note that get delegation token was called. Again this is hack for oozie
     * to make sure we add history server delegation tokens to the credentials
     */
    RMDelegationTokenSelector tokenSelector = new RMDelegationTokenSelector();
    Text service = resMgrDelegate.getRMDelegationTokenService();
    if (tokenSelector.selectToken(service, ts.getAllTokens()) != null) {
      Text hsService = SecurityUtil.buildTokenService(hsProxy
          .getConnectAddress());
      if (ts.getToken(hsService) == null) {
        ts.addToken(hsService, getDelegationTokenFromHS(hsProxy));
      }
    }
  }
}
 

import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; //导入依赖的package包/类
@Test
public void testRemoteExceptionFromHistoryServer() throws Exception {

  MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
  when(historyServerProxy.getJobReport(getJobReportRequest())).thenThrow(
      new IOException("Job ID doesnot Exist"));

  ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
  when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
      .thenReturn(null);

  ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(
      historyServerProxy, rm);

  try {
    clientServiceDelegate.getJobStatus(oldJobId);
    Assert.fail("Invoke should throw exception after retries.");
  } catch (IOException e) {
    Assert.assertTrue(e.getMessage().contains(
        "Job ID doesnot Exist"));
  }
}
 

import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; //导入依赖的package包/类
@Test
public void testRetriesOnConnectionFailure() throws Exception {

  MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);
  when(historyServerProxy.getJobReport(getJobReportRequest())).thenThrow(
      new RuntimeException("1")).thenThrow(new RuntimeException("2"))       
      .thenReturn(getJobReportResponse());

  ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);
  when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))
      .thenReturn(null);

  ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(
      historyServerProxy, rm);

  JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
  Assert.assertNotNull(jobStatus);
  verify(historyServerProxy, times(3)).getJobReport(
      any(GetJobReportRequest.class));
}
 

import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; //导入依赖的package包/类
@Test
public void testJobReportFromHistoryServer() throws Exception {                                 
  MRClientProtocol historyServerProxy = mock(MRClientProtocol.class);                           
  when(historyServerProxy.getJobReport(getJobReportRequest())).thenReturn(                      
      getJobReportResponseFromHistoryServer());                                                 
  ResourceMgrDelegate rm = mock(ResourceMgrDelegate.class);                                     
  when(rm.getApplicationReport(TypeConverter.toYarn(oldJobId).getAppId()))                      
  .thenReturn(null);                                                                        
  ClientServiceDelegate clientServiceDelegate = getClientServiceDelegate(                       
      historyServerProxy, rm);

  JobStatus jobStatus = clientServiceDelegate.getJobStatus(oldJobId);
  Assert.assertNotNull(jobStatus);
  Assert.assertEquals("TestJobFilePath", jobStatus.getJobFile());                               
  Assert.assertEquals("http://TestTrackingUrl", jobStatus.getTrackingUrl());                    
  Assert.assertEquals(1.0f, jobStatus.getMapProgress(), 0.0f);
  Assert.assertEquals(1.0f, jobStatus.getReduceProgress(), 0.0f);
}
 

import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; //导入依赖的package包/类
private Token getDelegationToken(
    final UserGroupInformation loggedInUser,
    final MRClientProtocol hsService, final String renewerString)
    throws IOException, InterruptedException {
  // Get the delegation token directly as it is a little difficult to setup
  // the kerberos based rpc.
  Token token = loggedInUser
      .doAs(new PrivilegedExceptionAction<Token>() {
        @Override
        public Token run() throws IOException {
          GetDelegationTokenRequest request = Records
              .newRecord(GetDelegationTokenRequest.class);
          request.setRenewer(renewerString);
          return hsService.getDelegationToken(request).getDelegationToken();
        }

      });
  return token;
}
 

import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; //导入依赖的package包/类
private MRClientProtocol getMRClientProtocol(Token token,
    final InetSocketAddress hsAddress, String user, final Configuration conf) {
  UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
  ugi.addToken(ConverterUtils.convertFromYarn(token, hsAddress));

  final YarnRPC rpc = YarnRPC.create(conf);
  MRClientProtocol hsWithDT = ugi
      .doAs(new PrivilegedAction<MRClientProtocol>() {

        @Override
        public MRClientProtocol run() {
          return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class,
              hsAddress, conf);
        }
      });
  return hsWithDT;
}
 

import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; //导入依赖的package包/类
@Override
public long renew(Token<?> token, Configuration conf) throws IOException,
    InterruptedException {

  org.apache.hadoop.yarn.api.records.Token dToken =
      org.apache.hadoop.yarn.api.records.Token.newInstance(
        token.getIdentifier(), token.getKind().toString(),
        token.getPassword(), token.getService().toString());

  MRClientProtocol histProxy = instantiateHistoryProxy(conf,
      SecurityUtil.getTokenServiceAddr(token));
  try {
    RenewDelegationTokenRequest request = Records
        .newRecord(RenewDelegationTokenRequest.class);
    request.setDelegationToken(dToken);
    return histProxy.renewDelegationToken(request).getNextExpirationTime();
  } finally {
    stopHistoryProxy(histProxy);
  }

}
 

import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; //导入依赖的package包/类
@Override
public void cancel(Token<?> token, Configuration conf) throws IOException,
    InterruptedException {

  org.apache.hadoop.yarn.api.records.Token dToken =
      org.apache.hadoop.yarn.api.records.Token.newInstance(
        token.getIdentifier(), token.getKind().toString(),
        token.getPassword(), token.getService().toString());

  MRClientProtocol histProxy = instantiateHistoryProxy(conf,
      SecurityUtil.getTokenServiceAddr(token));
  try {
    CancelDelegationTokenRequest request = Records
        .newRecord(CancelDelegationTokenRequest.class);
    request.setDelegationToken(dToken);
    histProxy.cancelDelegationToken(request);
  } finally {
    stopHistoryProxy(histProxy);
  }
}
 

import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; //导入依赖的package包/类
protected MRClientProtocol instantiateHistoryProxy(final Configuration conf,
    final InetSocketAddress hsAddress) throws IOException {

  if (LOG.isDebugEnabled()) {
    LOG.debug("Connecting to MRHistoryServer at: " + hsAddress);
  }
  final YarnRPC rpc = YarnRPC.create(conf);
  UserGroupInformation currentUser = UserGroupInformation.getCurrentUser();
  return currentUser.doAs(new PrivilegedAction<MRClientProtocol>() {
    @Override
    public MRClientProtocol run() {
      return (MRClientProtocol) rpc.getProxy(HSClientProtocol.class,
          hsAddress, conf);
    }
  });
}
 

import org.apache.hadoop.mapreduce.v2.api.MRClientProtocol; //导入依赖的package包/类
private void testPbServerFactory() {
  InetSocketAddress addr = new InetSocketAddress(0);
  Configuration conf = new Configuration();
  MRClientProtocol instance = new MRClientProtocolTestImpl();
  Server server = null;
  try {
    server = 
      RpcServerFactoryPBImpl.get().getServer(
        MRClientProtocol.class, instance, addr, conf, null, 1);
    server.start();
  } catch (YarnRuntimeException e) {
    e.printStackTrace();
    Assert.fail("Failed to crete server");
  } finally {
    server.stop();
  }
}