Java AMRMClientAsync.CallbackHandler方法代码示例

import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; //导入方法依赖的package包/类
/**
 * Register RM callback and start listening
 */
private void registerRMCallBackHandler() {
    AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
    setAmRMClient(AMRMClientAsync.createAMRMClientAsync(1000, allocListener));
    getAmRMClient().init(getYarnConf());
    getAmRMClient().start();
}
 

import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; //导入方法依赖的package包/类
public ClusterScheduler(AppContext appContext, String trackingUrl, int rpcPort) {
  AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
  this.amRmClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
  this.containerListener = new NMCallbackHandler(this);
  this.nmClientAsync = new NMClientAsyncImpl(containerListener);
  this.appContext = appContext;
  this.appMasterTrackingUrl = trackingUrl;
  this.appMasterRpcPort = rpcPort;
}
 

import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; //导入方法依赖的package包/类
public boolean run() throws IOException, YarnException {
  // Initialize clients to RM and NMs.
  LOG.info("ApplicationMaster::run");
  LOG.error("command: " + this.command);
  AMRMClientAsync.CallbackHandler rmListener = new RMCallbackHandler();
  resourceManager = AMRMClientAsync.createAMRMClientAsync(1000, rmListener);
  resourceManager.init(conf);
  resourceManager.start();

  nodeManager = NMClient.createNMClient();
  nodeManager.init(conf);
  nodeManager.start();

  // Register with RM
  resourceManager.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl);


  // Ask RM to give us a bunch of containers
  for (int i = 0; i < totalContainerCount; i++) {
    ContainerRequest containerReq = setupContainerReqForRM();
    resourceManager.addContainerRequest(containerReq);
  }
  requestedContainerCount.addAndGet(totalContainerCount);

  while (!done) {
    try {
      Thread.sleep(200);
    } catch (InterruptedException ex) {
    }
  }// while

  // Un-register with ResourceManager
  resourceManager.unregisterApplicationMaster( FinalApplicationStatus.SUCCEEDED, "", "");
  return true;
}
 

import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; //导入方法依赖的package包/类
public boolean run() throws IOException, YarnException {
  // Initialize clients to RM and NMs.
  LOG.info("ApplicationMaster::run");
  AMRMClientAsync.CallbackHandler rmListener = new RMCallbackHandler();
  resourceManager = AMRMClientAsync.createAMRMClientAsync(1000, rmListener);
  resourceManager.init(conf);
  resourceManager.start();
  
  nodeManager = NMClient.createNMClient();
  nodeManager.init(conf);
  nodeManager.start();
  
  // Register with RM
  resourceManager.registerApplicationMaster(appMasterHostname, appMasterRpcPort,
      appMasterTrackingUrl);

  Log.info("total container count: "+Integer.toString(totalContainerCount));
  
  // Ask RM to give us a bunch of containers
  //for (int i = 0; i < totalContainerCount; i++) {
    ContainerRequest containerReq = setupContainerReqForRM();
    resourceManager.addContainerRequest(containerReq);
  //}
  requestedContainerCount.addAndGet(totalContainerCount);

  while (!done) {
    try {
      Thread.sleep(200);
    } catch (InterruptedException ex) {
    }
  }// while

  // Un-register with ResourceManager
  resourceManager.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");
  return true;
}
 

import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; //导入方法依赖的package包/类
/**
 * Main run function for the application master
 *
 * @throws YarnException
 * @throws IOException
 */
@SuppressWarnings({ "unchecked" })
public boolean run() throws YarnException, IOException {
  LOG.info("Starting ApplicationMaster");

  AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
  resourceManager =
      AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
  resourceManager.init(conf);
  resourceManager.start();

  containerListener = new NMCallbackHandler();
  nmClientAsync = new NMClientAsyncImpl(containerListener);
  nmClientAsync.init(conf);
  nmClientAsync.start();

  // Setup local RPC Server to accept status requests directly from clients
  // TODO need to setup a protocol for client to be able to communicate to
  // the RPC server
  // TODO use the rpc port info to register with the RM for the client to
  // send requests to this app master

  // Register self with ResourceManager
  // This will start heartbeating to the RM
  RegisterApplicationMasterResponse response = resourceManager
      .registerApplicationMaster(appMasterHostname, appMasterRpcPort,
          appMasterTrackingUrl);
  // Dump out information about cluster capability as seen by the
  // resource manager
  int maxMem = response.getMaximumResourceCapability().getMemory();
  LOG.info("Max mem capabililty of resources in this cluster " + maxMem);

  // A resource ask cannot exceed the max.
  if (containerMemory > maxMem) {
    LOG.info("Container memory specified above max threshold of cluster."
        + " Using max value." + ", specified=" + containerMemory + ", max="
        + maxMem);
    containerMemory = maxMem;
  }


  // Setup ask for containers from RM
  // Send request for containers to RM
  // Until we get our fully allocated quota, we keep on polling RM for
  // containers
  // Keep looping until all the containers are launched and shell script
  // executed on them ( regardless of success/failure).
  for (int i = 0; i < numTotalContainers; ++i) {
    ContainerRequest containerAsk = setupContainerAskForRM();
    resourceManager.addContainerRequest(containerAsk);
  }
  numRequestedContainers.set(numTotalContainers);

  while (!done) {
    try {
      Thread.sleep(200);
    } catch (InterruptedException ex) {}
  }
  finish();
  
  return success;
}
 

import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; //导入方法依赖的package包/类
/**
 * Main run function for the application master
 *
 * @throws YarnException
 * @throws IOException
 */
@SuppressWarnings({ "unchecked" })
public boolean run() throws YarnException, IOException {
  LOG.info("Starting ApplicationMaster");

  Credentials credentials =
      UserGroupInformation.getCurrentUser().getCredentials();
  DataOutputBuffer dob = new DataOutputBuffer();
  credentials.writeTokenStorageToStream(dob);
  // Now remove the AM->RM token so that containers cannot access it.
  Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
  while (iter.hasNext()) {
    Token<?> token = iter.next();
    if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
      iter.remove();
    }
  }
  allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

  AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
  amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
  amRMClient.init(conf);
  amRMClient.start();

  containerListener = createNMCallbackHandler();
  nmClientAsync = new NMClientAsyncImpl(containerListener);
  nmClientAsync.init(conf);
  nmClientAsync.start();

  // Setup local RPC Server to accept status requests directly from clients
  // TODO need to setup a protocol for client to be able to communicate to
  // the RPC server
  // TODO use the rpc port info to register with the RM for the client to
  // send requests to this app master

  // Register self with ResourceManager
  // This will start heartbeating to the RM
  appMasterHostname = NetUtils.getHostname();
  RegisterApplicationMasterResponse response = amRMClient
      .registerApplicationMaster(appMasterHostname, appMasterRpcPort,
          appMasterTrackingUrl);
  // Dump out information about cluster capability as seen by the
  // resource manager
  int maxMem = response.getMaximumResourceCapability().getMemory();
  LOG.info("Max mem capabililty of resources in this cluster " + maxMem);

  // A resource ask cannot exceed the max.
  if (containerMemory > maxMem) {
    LOG.info("Container memory specified above max threshold of cluster."
        + " Using max value." + ", specified=" + containerMemory + ", max="
        + maxMem);
    containerMemory = maxMem;
  }

  // Setup ask for containers from RM
  // Send request for containers to RM
  // Until we get our fully allocated quota, we keep on polling RM for
  // containers
  // Keep looping until all the containers are launched and shell script
  // executed on them ( regardless of success/failure).
  for (int i = 0; i < numTotalContainers; ++i) {
    ContainerRequest containerAsk = setupContainerAskForRM();
    amRMClient.addContainerRequest(containerAsk);
  }
  numRequestedContainers.set(numTotalContainers);

  while (!done
      && (numCompletedContainers.get() != numTotalContainers)) {
    try {
      Thread.sleep(200);
    } catch (InterruptedException ex) {}
  }
  finish();
  
  return success;
}
 

import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; //导入方法依赖的package包/类
/**
 * Main run function for the application master
 * 
 * @throws YarnException
 * @throws IOException
 */
@SuppressWarnings({ "unchecked" })
public boolean run() throws YarnException, IOException {
	LOG.info("Starting JBossApplicationMaster");

	AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
	resourceManager = AMRMClientAsync.createAMRMClientAsync(1000,
			allocListener);
	resourceManager.init(conf);
	resourceManager.start();

	containerListener = new NMCallbackHandler();
	nmClientAsync = new NMClientAsyncImpl(containerListener);
	nmClientAsync.init(conf);
	nmClientAsync.start();

	RegisterApplicationMasterResponse response = resourceManager
			.registerApplicationMaster(appMasterHostname, appMasterRpcPort,
					appMasterTrackingUrl);

	int maxMem = response.getMaximumResourceCapability().getMemory();
	LOG.info("Max mem capabililty of resources in this cluster " + maxMem);

	// A resource ask cannot exceed the max.
	if (containerMemory > maxMem) {
		LOG.info("Container memory specified above max threshold of cluster."
				+ " Using max value."
				+ ", specified="
				+ containerMemory
				+ ", max=" + maxMem);
		containerMemory = maxMem;
	}

	for (int i = 0; i < numTotalContainers; ++i) {
		ContainerRequest containerAsk = setupContainerAskForRM();
		resourceManager.addContainerRequest(containerAsk);
	}
	numRequestedContainers.set(numTotalContainers);

	while (!done) {
		try {
			Thread.sleep(200);
		} catch (InterruptedException ex) {
		}
	}
	finish();

	return success;
}
 

import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; //导入方法依赖的package包/类
public TezAMRMClientAsync(
    AMRMClient<T> client,
    int intervalMs,
    AMRMClientAsync.CallbackHandler callbackHandler) {
  super(client, intervalMs, callbackHandler);
}