Java NMClientAsync.createNMClientAsync方法代码示例

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入方法依赖的package包/类
public void init(ContainerLaunchContextFactory factory) throws IOException {
  this.nodeManager = NMClientAsync.createNMClientAsync(this);
  nodeManager.init(conf);
  nodeManager.start();

  this.ctxt = factory.create(params);
  LOG.info("Init CTXT: " + ctxt.getLocalResources());
  LOG.info("Init CTXT: " + ctxt.getCommands());
  
  this.resource = factory.createResource(params);
  this.priority = factory.createPriority(params.getPriority());
  AMRMClient.ContainerRequest containerRequest = new AMRMClient.ContainerRequest(
      resource,
      null, // nodes
      null, // racks
      priority);
  int numInstances = params.getNumInstances();
  for (int j = 0; j < numInstances; j++) {
    resourceManager.addContainerRequest(containerRequest);
  }
  needed.set(numInstances);
}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入方法依赖的package包/类
public void init(ContainerLaunchContextFactory factory) throws IOException {
  this.nodeManager = NMClientAsync.createNMClientAsync(this);
  nodeManager.init(conf);
  nodeManager.start();

  this.ctxt = factory.create(parameters);
  this.resource = factory.createResource(parameters);
  this.priority = factory.createPriority(parameters.getPriority());
  AMRMClient.ContainerRequest containerRequest = new AMRMClient.ContainerRequest(
      resource,
      null, // nodes
      null, // racks
      priority);
  int numInstances = parameters.getNumInstances();
  for (int j = 0; j < numInstances; j++) {
    resourceManager.addContainerRequest(containerRequest);
  }
  needed.set(numInstances);
  totalRequested.addAndGet(numInstances);
}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入方法依赖的package包/类
@Override
protected void initialize() throws Exception {
  super.initialize();
  transitionTaskStateQueue = new SystemTaskQueue(this::handleException);

  // Initialize AM NoDependenceConfig
  conf.initializeNoDependenceConfig();

  // Start RMClient to Register AM ASAP in case AM expired by RM
  rmClient = AMRMClientAsync.createAMRMClientAsync(
      conf.getAmRmHeartbeatIntervalSec() * 1000,
      new RMClientCallbackHandler(this));
  rmClient.init(conf.getYarnConfig());
  rmClient.start();
  conf.initializeDependOnRMResponseConfig(registerToRM());

  // Start NMClient
  nmClient = NMClientAsync.createNMClientAsync(
      new NMClientCallbackHandler(this));
  nmClient.init(conf.getYarnConfig());
  nmClient.start();

  // Start YarnClient
  yarnClient = YarnClient.createYarnClient();
  yarnClient.init(conf.getYarnConfig());
  yarnClient.start();
  conf.initializeDependOnYarnClientConfig(yarnClient);

  // Initialize Launcher Store
  zkStore = new ZookeeperStore(conf.getZkConnectString(), conf.getZkRootDir());
  conf.initializeDependOnZKStoreConfig(zkStore);
  hdfsStore = new HdfsStore(conf.getLauncherConfig().getHdfsRootDir());
  hdfsStore.makeFrameworkRootDir(conf.getFrameworkName());
  hdfsStore.makeAMStoreRootDir(conf.getFrameworkName());

  // Initialize other components
  launcherClient = new LauncherClientInternal(
      conf.getLauncherConfig().getWebServerAddress(), 30, 10,
      LaunchClientType.APPLICATION_MASTER);

  gpuAllocationManager = new GpuAllocationManager(this);
  rmResyncHandler = new RMResyncHandler(this, conf);
}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入方法依赖的package包/类
protected NMClientAsync createNMClientAsync() {
  launchListener = new NMCallback();
  nmClientAsync = NMClientAsync.createNMClientAsync(launchListener);
  nmClientAsync.init(context.getConf());
  return nmClientAsync;
}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入方法依赖的package包/类
public void init(ContainerLaunchContextFactory factory) throws IOException {
  LOG.info( "Are all previous containers finished?");
  if(!allPreviousFinished()){
	  LOG.info( "They are not all previous containers finished.");
	  return;
  }
  LOG.info( "They are all previous containers finished.");
  service.parameters.workflow.getOperator(params.getName()).setStatus("running");
  startTime = System.currentTimeMillis();
  this.nodeManager = NMClientAsync.createNMClientAsync(this);
  nodeManager.init(service.conf);
  nodeManager.start();
  isInitialized=true;
  
  this.resource = factory.createResource(params);

  //this.priority = factory.createPriority(params.getPriority());
  
  //hack for https://issues.apache.org/jira/browse/YARN-314
  this.priority = factory.createPriority(service.prior);
  service.prior++;
  //hack for https://issues.apache.org/jira/browse/YARN-314
  
  int numInstances = params.getNumInstances();
  LOG.info("Operator: "+params.getName()+" requesting " + numInstances+" containers");
  LOG.info("Resource cores: "+ resource.getVirtualCores());
  LOG.info("Resource memory: "+ resource.getMemory());
  String[] nodes =params.getNodes();//= {"slave1"};
  String labels = params.getLabels();
  AMRMClient.ContainerRequest containerRequest=null;
  if(labels==null){
	  LOG.info("Resource nodes: all");
  	  containerRequest = new AMRMClient.ContainerRequest(
          resource,
          nodes, // nodes
          null, // racks
          priority,
          true, //true for relaxed locality
          "");
  }
  else{
	  LOG.info("Resource labels: "+ labels);
	  for (int i = 0; i < nodes.length; i++) {
    	  LOG.info("Resource nodes: "+ nodes[i]);
	  }
	  containerRequest = new AMRMClient.ContainerRequest(
	          resource,
	          nodes, // nodes
	          null, // racks
	          priority,
	          false, //true for relaxed locality
	          "");
  }
  LOG.info( "Container request is: " + containerRequest);
  this.containerRequests = new ArrayList<AMRMClient.ContainerRequest>();
  //restartResourceManager();
  for (int j = 0; j < numInstances; j++) {
	  service.resourceManager.addContainerRequest(containerRequest);
	  containerRequests.add(containerRequest);
  }
  LOG.info( "NumInstances is: " + numInstances);
  LOG.info( "ResourceManager: " + service.resourceManager);
  
  needed.set(numInstances);
}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入方法依赖的package包/类
/**
 * Creates an YarnClusterResourceManager from config, a jobModelReader and a callback.
 * @param config to instantiate the container manager with
 * @param jobModelManager the jobModel manager to get the job model (mostly for the UI)
 * @param callback the callback to receive events from Yarn.
 * @param samzaAppState samza app state for display in the UI
 */
public YarnClusterResourceManager(Config config, JobModelManager jobModelManager, ClusterResourceManager.Callback callback, SamzaApplicationState samzaAppState ) {
  super(callback);
  yarnConfiguration = new YarnConfiguration();
  yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());

  // Use the Samza job config "fs.<scheme>.impl" and "fs.<scheme>.impl.*" for YarnConfiguration
  FileSystemImplConfig fsImplConfig = new FileSystemImplConfig(config);
  fsImplConfig.getSchemes().forEach(
      scheme -> {
        fsImplConfig.getSchemeConfig(scheme).forEach(
            (confKey, confValue) -> yarnConfiguration.set(confKey, confValue)
        );
      }
  );

  MetricsRegistryMap registry = new MetricsRegistryMap();
  metrics = new SamzaAppMasterMetrics(config, samzaAppState, registry);

  // parse configs from the Yarn environment
  String containerIdStr = System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString());
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  String nodeHostString = System.getenv(ApplicationConstants.Environment.NM_HOST.toString());
  String nodePortString = System.getenv(ApplicationConstants.Environment.NM_PORT.toString());
  String nodeHttpPortString = System.getenv(ApplicationConstants.Environment.NM_HTTP_PORT.toString());

  int nodePort = Integer.parseInt(nodePortString);
  int nodeHttpPort = Integer.parseInt(nodeHttpPortString);
  YarnConfig yarnConfig = new YarnConfig(config);
  this.yarnConfig = yarnConfig;
  this.config = config;
  int interval = yarnConfig.getAMPollIntervalMs();

  //Instantiate the AM Client.
  this.amClient = AMRMClientAsync.createAMRMClientAsync(interval, this);

  this.state = new YarnAppState(-1, containerId, nodeHostString, nodePort, nodeHttpPort);

  log.info("Initialized YarnAppState: {}", state.toString());
  this.service = new SamzaYarnAppMasterService(config, samzaAppState, this.state, registry, yarnConfiguration);

  log.info("ContainerID str {}, Nodehost  {} , Nodeport  {} , NodeHttpport {}", new Object [] {containerIdStr, nodeHostString, nodePort, nodeHttpPort});
  ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config);
  this.lifecycle = new SamzaYarnAppMasterLifecycle(
      clusterManagerConfig.getContainerMemoryMb(),
      clusterManagerConfig.getNumCores(),
      samzaAppState,
      state,
      amClient
  );
  this.nmClientAsync = NMClientAsync.createNMClientAsync(this);

}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入方法依赖的package包/类
public void run() throws YarnException, IOException {
	amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, new RMCallbackHandler());
	amRMClient.init(conf);
	amRMClient.start();
	
	RegisterApplicationMasterResponse response;
	response = amRMClient.registerApplicationMaster(NetUtils.getHostname(), -1, "");
	LOG.info("ApplicationMaster is registered with response: {}", response.toString());
	
	containerListener = new NMCallbackHandler(this);
	nmClient = NMClientAsync.createNMClientAsync(containerListener);
	nmClient.init(conf);
	nmClient.start();
	
	Resource capacity = Records.newRecord(Resource.class);
	capacity.setMemory(128);
	Priority priority = Records.newRecord(Priority.class);
	priority.setPriority(0);
	for(int i = 1; i <= 5; i++) {
		ContainerRequest ask = new ContainerRequest(capacity,null, null,priority);
		amRMClient.addContainerRequest(ask);
		numOfContainers++;
	}
	
	while(!done && numCompletedContainers.get() < numOfContainers) {
		LOG.info("The number of completed Containers = " +
				this.numCompletedContainers.get());
		try {
			Thread.sleep(2000);
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
	}
	
	LOG.info("Containers have all completed, so shutting down NMClient and AMRMClient...");

	nmClient.stop();
	amRMClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "Application complete!", null);
	amRMClient.stop();
	
}