Java NMClientAsyncImpl类代码示例

import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; //导入依赖的package包/类
/**
 * Register NM callback and start listening
 */
private void registerNMCallbackHandler() {
    setContainerListener(new NMCallbackHandler());
    setNmClientAsync(new NMClientAsyncImpl(getContainerListener()));
    getNmClientAsync().init(getYarnConf());
    getNmClientAsync().start();
}
 

import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; //导入依赖的package包/类
public ClusterScheduler(AppContext appContext, String trackingUrl, int rpcPort) {
  AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
  this.amRmClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
  this.containerListener = new NMCallbackHandler(this);
  this.nmClientAsync = new NMClientAsyncImpl(containerListener);
  this.appContext = appContext;
  this.appMasterTrackingUrl = trackingUrl;
  this.appMasterRpcPort = rpcPort;
}
 

import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; //导入依赖的package包/类
@Inject
private YarnContainerManager(
    @Parameter(YarnHeartbeatPeriod.class) final int yarnRMHeartbeatPeriod,
    @Parameter(JobSubmissionDirectory.class) final String jobSubmissionDirectory,
    final YarnConfiguration yarnConf,
    final YarnProxyUser yarnProxyUser,
    final REEFEventHandlers reefEventHandlers,
    final Containers containers,
    final ApplicationMasterRegistration registration,
    final ContainerRequestCounter containerRequestCounter,
    final DriverStatusManager driverStatusManager,
    final REEFFileNames reefFileNames,
    final TrackingURLProvider trackingURLProvider,
    final LocalAddressProvider addressProvider,
    final RackNameFormatter rackNameFormatter,
    final InjectionFuture<ProgressProvider> progressProvider) throws IOException {

  this.reefEventHandlers = reefEventHandlers;
  this.driverStatusManager = driverStatusManager;

  this.containers = containers;
  this.registration = registration;
  this.containerRequestCounter = containerRequestCounter;
  this.yarnConf = yarnConf;
  this.yarnProxyUser = yarnProxyUser;
  this.rackNameFormatter = rackNameFormatter;

  this.trackingUrl = trackingURLProvider.getTrackingUrl();
  this.amRegistrationHost = addressProvider.getLocalAddress();

  this.resourceManager = AMRMClientAsync.createAMRMClientAsync(yarnRMHeartbeatPeriod, this);
  this.nodeManager = new NMClientAsyncImpl(this);

  this.jobSubmissionDirectory = jobSubmissionDirectory;
  this.reefFileNames = reefFileNames;
  this.progressProvider = progressProvider;

  LOG.log(Level.INFO, "Instantiated YarnContainerManager: {0} {1}, trackingUrl: {3}, jobSubmissionDirectory: {4}.",
      new Object[] {this.registration, this.yarnProxyUser, this.trackingUrl, this.jobSubmissionDirectory});
}
 

import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; //导入依赖的package包/类
private void setupNMConnection() {
  containerListener = new NMCallbackHandler(this);
  nmClientAsync = new NMClientAsyncImpl(containerListener);
  nmClientAsync.init(conf);
  nmClientAsync.start();
}
 

import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; //导入依赖的package包/类
public static NMClientAsync createNMClientAsync(
    CallbackHandler callbackHandler) {
  return new NMClientAsyncImpl(callbackHandler);
}
 

import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; //导入依赖的package包/类
public static NMClientAsync createNMClientAsync(
    AbstractCallbackHandler callbackHandler) {
  return new NMClientAsyncImpl(callbackHandler);
}
 

import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; //导入依赖的package包/类
@Override
protected void serviceInit(Configuration conf) throws Exception
{
  LOG.info("Application master" + ", appId=" + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" + appAttemptID.getApplicationId().getClusterTimestamp() + ", attemptId=" + appAttemptID.getAttemptId());

  FileInputStream fis = new FileInputStream("./" + LogicalPlan.SER_FILE_NAME);
  try {
    this.dag = LogicalPlan.read(fis);
  } finally {
    fis.close();
  }
  // "debug" simply dumps all data using LOG.info
  if (dag.isDebug()) {
    dumpOutDebugInfo();
  }
  dag.setAttribute(LogicalPlan.APPLICATION_ATTEMPT_ID, appAttemptID.getAttemptId());
  FSRecoveryHandler recoveryHandler = new FSRecoveryHandler(dag.assertAppPath(), conf);
  this.dnmgr = StreamingContainerManager.getInstance(recoveryHandler, dag, true);
  dag = this.dnmgr.getLogicalPlan();
  this.appContext = new ClusterAppContextImpl(dag.getAttributes());

  Map<Class<?>, Class<? extends StringCodec<?>>> codecs = dag.getAttributes().get(DAG.STRING_CODECS);
  StringCodecs.loadConverters(codecs);

  LOG.info("Starting application with {} operators in {} containers", dnmgr.getPhysicalPlan().getAllOperators().size(), dnmgr.getPhysicalPlan().getContainers().size());

  // Setup security configuration such as that for web security
  SecurityUtils.init(conf, dag.getValue(LogicalPlan.STRAM_HTTP_AUTHENTICATION));

  if (UserGroupInformation.isSecurityEnabled()) {
    // TODO :- Need to perform token renewal
    delegationTokenManager = new StramDelegationTokenManager(DELEGATION_KEY_UPDATE_INTERVAL, DELEGATION_TOKEN_MAX_LIFETIME, DELEGATION_TOKEN_RENEW_INTERVAL, DELEGATION_TOKEN_REMOVER_SCAN_INTERVAL);
  }
  this.nmClient = new NMClientAsyncImpl(new NMCallbackHandler());
  addService(nmClient);
  this.amRmClient = AMRMClient.createAMRMClient();
  addService(amRmClient);

  // start RPC server
  int rpcListenerCount = dag.getValue(DAGContext.HEARTBEAT_LISTENER_THREAD_COUNT);
  this.heartbeatListener = new StreamingContainerParent(this.getClass().getName(), dnmgr, delegationTokenManager, rpcListenerCount);
  addService(heartbeatListener);

  AutoMetric.Transport appDataPushTransport = dag.getValue(LogicalPlan.METRICS_TRANSPORT);
  if (appDataPushTransport != null) {
    this.appDataPushAgent = new AppDataPushAgent(dnmgr, appContext);
    addService(this.appDataPushAgent);
  }
  initApexPluginDispatcher();

  // Initialize all services added above
  super.serviceInit(conf);
}
 

import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; //导入依赖的package包/类
/**
 * Main run function for the application master
 *
 * @throws YarnException
 * @throws IOException
 */
@SuppressWarnings({ "unchecked" })
public boolean run() throws YarnException, IOException {
  LOG.info("Starting ApplicationMaster");

  AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
  resourceManager =
      AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
  resourceManager.init(conf);
  resourceManager.start();

  containerListener = new NMCallbackHandler();
  nmClientAsync = new NMClientAsyncImpl(containerListener);
  nmClientAsync.init(conf);
  nmClientAsync.start();

  // Setup local RPC Server to accept status requests directly from clients
  // TODO need to setup a protocol for client to be able to communicate to
  // the RPC server
  // TODO use the rpc port info to register with the RM for the client to
  // send requests to this app master

  // Register self with ResourceManager
  // This will start heartbeating to the RM
  RegisterApplicationMasterResponse response = resourceManager
      .registerApplicationMaster(appMasterHostname, appMasterRpcPort,
          appMasterTrackingUrl);
  // Dump out information about cluster capability as seen by the
  // resource manager
  int maxMem = response.getMaximumResourceCapability().getMemory();
  LOG.info("Max mem capabililty of resources in this cluster " + maxMem);

  // A resource ask cannot exceed the max.
  if (containerMemory > maxMem) {
    LOG.info("Container memory specified above max threshold of cluster."
        + " Using max value." + ", specified=" + containerMemory + ", max="
        + maxMem);
    containerMemory = maxMem;
  }


  // Setup ask for containers from RM
  // Send request for containers to RM
  // Until we get our fully allocated quota, we keep on polling RM for
  // containers
  // Keep looping until all the containers are launched and shell script
  // executed on them ( regardless of success/failure).
  for (int i = 0; i < numTotalContainers; ++i) {
    ContainerRequest containerAsk = setupContainerAskForRM();
    resourceManager.addContainerRequest(containerAsk);
  }
  numRequestedContainers.set(numTotalContainers);

  while (!done) {
    try {
      Thread.sleep(200);
    } catch (InterruptedException ex) {}
  }
  finish();
  
  return success;
}
 

import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; //导入依赖的package包/类
/**
 * Main run function for the application master
 *
 * @throws YarnException
 * @throws IOException
 */
@SuppressWarnings({ "unchecked" })
public boolean run() throws YarnException, IOException {
  LOG.info("Starting ApplicationMaster");

  Credentials credentials =
      UserGroupInformation.getCurrentUser().getCredentials();
  DataOutputBuffer dob = new DataOutputBuffer();
  credentials.writeTokenStorageToStream(dob);
  // Now remove the AM->RM token so that containers cannot access it.
  Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
  while (iter.hasNext()) {
    Token<?> token = iter.next();
    if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
      iter.remove();
    }
  }
  allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

  AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
  amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
  amRMClient.init(conf);
  amRMClient.start();

  containerListener = createNMCallbackHandler();
  nmClientAsync = new NMClientAsyncImpl(containerListener);
  nmClientAsync.init(conf);
  nmClientAsync.start();

  // Setup local RPC Server to accept status requests directly from clients
  // TODO need to setup a protocol for client to be able to communicate to
  // the RPC server
  // TODO use the rpc port info to register with the RM for the client to
  // send requests to this app master

  // Register self with ResourceManager
  // This will start heartbeating to the RM
  appMasterHostname = NetUtils.getHostname();
  RegisterApplicationMasterResponse response = amRMClient
      .registerApplicationMaster(appMasterHostname, appMasterRpcPort,
          appMasterTrackingUrl);
  // Dump out information about cluster capability as seen by the
  // resource manager
  int maxMem = response.getMaximumResourceCapability().getMemory();
  LOG.info("Max mem capabililty of resources in this cluster " + maxMem);

  // A resource ask cannot exceed the max.
  if (containerMemory > maxMem) {
    LOG.info("Container memory specified above max threshold of cluster."
        + " Using max value." + ", specified=" + containerMemory + ", max="
        + maxMem);
    containerMemory = maxMem;
  }

  // Setup ask for containers from RM
  // Send request for containers to RM
  // Until we get our fully allocated quota, we keep on polling RM for
  // containers
  // Keep looping until all the containers are launched and shell script
  // executed on them ( regardless of success/failure).
  for (int i = 0; i < numTotalContainers; ++i) {
    ContainerRequest containerAsk = setupContainerAskForRM();
    amRMClient.addContainerRequest(containerAsk);
  }
  numRequestedContainers.set(numTotalContainers);

  while (!done
      && (numCompletedContainers.get() != numTotalContainers)) {
    try {
      Thread.sleep(200);
    } catch (InterruptedException ex) {}
  }
  finish();
  
  return success;
}
 

import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; //导入依赖的package包/类
/**
 * Main run function for the application master
 * 
 * @throws YarnException
 * @throws IOException
 */
@SuppressWarnings({ "unchecked" })
public boolean run() throws YarnException, IOException {
	LOG.info("Starting JBossApplicationMaster");

	AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
	resourceManager = AMRMClientAsync.createAMRMClientAsync(1000,
			allocListener);
	resourceManager.init(conf);
	resourceManager.start();

	containerListener = new NMCallbackHandler();
	nmClientAsync = new NMClientAsyncImpl(containerListener);
	nmClientAsync.init(conf);
	nmClientAsync.start();

	RegisterApplicationMasterResponse response = resourceManager
			.registerApplicationMaster(appMasterHostname, appMasterRpcPort,
					appMasterTrackingUrl);

	int maxMem = response.getMaximumResourceCapability().getMemory();
	LOG.info("Max mem capabililty of resources in this cluster " + maxMem);

	// A resource ask cannot exceed the max.
	if (containerMemory > maxMem) {
		LOG.info("Container memory specified above max threshold of cluster."
				+ " Using max value."
				+ ", specified="
				+ containerMemory
				+ ", max=" + maxMem);
		containerMemory = maxMem;
	}

	for (int i = 0; i < numTotalContainers; ++i) {
		ContainerRequest containerAsk = setupContainerAskForRM();
		resourceManager.addContainerRequest(containerAsk);
	}
	numRequestedContainers.set(numTotalContainers);

	while (!done) {
		try {
			Thread.sleep(200);
		} catch (InterruptedException ex) {
		}
	}
	finish();

	return success;
}
 

import org.apache.hadoop.yarn.client.api.async.impl.NMClientAsyncImpl; //导入依赖的package包/类
@Test
public void testAmShutdown() throws IOException, YarnException {

  Assume.assumeTrue(
      "This test requires a YARN Resource Manager to connect to",
      Boolean.parseBoolean(System.getenv("REEF_TEST_YARN")));

  final YarnConfiguration yarnConfig = new YarnConfiguration();

  // Start YARN client and register the application

  final YarnClient yarnClient = YarnClient.createYarnClient();
  yarnClient.init(yarnConfig);
  yarnClient.start();

  final ContainerLaunchContext containerContext = Records.newRecord(ContainerLaunchContext.class);
  containerContext.setCommands(Collections.<String>emptyList());
  containerContext.setLocalResources(Collections.<String, LocalResource>emptyMap());
  containerContext.setEnvironment(Collections.<String, String>emptyMap());
  containerContext.setTokens(getTokens());

  final ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext();
  appContext.setApplicationName("REEF_Unmanaged_AM_Test");
  appContext.setAMContainerSpec(containerContext);
  appContext.setUnmanagedAM(true);
  appContext.setQueue("default");

  final ApplicationId applicationId = appContext.getApplicationId();
  LOG.log(Level.INFO, "Registered YARN application: {0}", applicationId);

  yarnClient.submitApplication(appContext);

  LOG.log(Level.INFO, "YARN application submitted: {0}", applicationId);

  addToken(yarnClient.getAMRMToken(applicationId));

  // Start the AM

  final AMRMClientAsync<AMRMClient.ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, this);
  rmClient.init(yarnConfig);
  rmClient.start();

  final NMClientAsync nmClient = new NMClientAsyncImpl(this);
  nmClient.init(yarnConfig);
  nmClient.start();

  final RegisterApplicationMasterResponse registration =
      rmClient.registerApplicationMaster(NetUtils.getHostname(), -1, null);

  LOG.log(Level.INFO, "Unmanaged AM is running: {0}", registration);

  rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "Success!", null);

  LOG.log(Level.INFO, "Unregistering AM: state {0}", rmClient.getServiceState());

  // Shutdown the AM

  rmClient.stop();
  nmClient.stop();

  // Get the final application report

  final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);
  final YarnApplicationState appState = appReport.getYarnApplicationState();
  final FinalApplicationStatus finalAttemptStatus = appReport.getFinalApplicationStatus();

  LOG.log(Level.INFO, "Application {0} final attempt {1} status: {2}/{3}", new Object[] {
      applicationId, appReport.getCurrentApplicationAttemptId(), appState, finalAttemptStatus});

  Assert.assertEquals("Application must be in FINISHED state", YarnApplicationState.FINISHED, appState);
  Assert.assertEquals("Final status must be SUCCEEDED", FinalApplicationStatus.SUCCEEDED, finalAttemptStatus);

  // Shutdown YARN client

  yarnClient.stop();
}