Java NMClientAsync类代码示例

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入依赖的package包/类
public void init(ContainerLaunchContextFactory factory) throws IOException {
  this.nodeManager = NMClientAsync.createNMClientAsync(this);
  nodeManager.init(conf);
  nodeManager.start();

  this.ctxt = factory.create(params);
  LOG.info("Init CTXT: " + ctxt.getLocalResources());
  LOG.info("Init CTXT: " + ctxt.getCommands());
  
  this.resource = factory.createResource(params);
  this.priority = factory.createPriority(params.getPriority());
  AMRMClient.ContainerRequest containerRequest = new AMRMClient.ContainerRequest(
      resource,
      null, // nodes
      null, // racks
      priority);
  int numInstances = params.getNumInstances();
  for (int j = 0; j < numInstances; j++) {
    resourceManager.addContainerRequest(containerRequest);
  }
  needed.set(numInstances);
}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入依赖的package包/类
public void init(ContainerLaunchContextFactory factory) throws IOException {
  this.nodeManager = NMClientAsync.createNMClientAsync(this);
  nodeManager.init(conf);
  nodeManager.start();

  this.ctxt = factory.create(parameters);
  this.resource = factory.createResource(parameters);
  this.priority = factory.createPriority(parameters.getPriority());
  AMRMClient.ContainerRequest containerRequest = new AMRMClient.ContainerRequest(
      resource,
      null, // nodes
      null, // racks
      priority);
  int numInstances = parameters.getNumInstances();
  for (int j = 0; j < numInstances; j++) {
    resourceManager.addContainerRequest(containerRequest);
  }
  needed.set(numInstances);
  totalRequested.addAndGet(numInstances);
}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入依赖的package包/类
public LaunchContainer( Configuration conf
        , String zkQuorum
        , String zkRoot
        , NMClientAsync nmClientAsync
        , ModelRequest request
        , Container container
        , ByteBuffer allTokens
        , Path appJarLocation
)
{
  this.allTokens = allTokens;
  this.zkQuorum = zkQuorum;
  this.zkRoot = zkRoot;
  this.request = request;
  this.container = container;
  this.conf = conf;
  this.nmClientAsync = nmClientAsync;
  this.appJarLocation = appJarLocation;
}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入依赖的package包/类
public StatefulContainer(NMClientAsync client, ContainerId containerId) {
  this.nmClientAsync = client;
  this.containerId = containerId;
  stateMachine = stateMachineFactory.make(this);
  ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
  readLock = lock.readLock();
  writeLock = lock.writeLock();
}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入依赖的package包/类
/**
 * @param lcontainer Allocated container
 * @param nmClient
 * @param sca
 * @param tokens
 */
public LaunchContainerRunnable(Container lcontainer, NMClientAsync nmClient, StreamingContainerAgent sca, ByteBuffer tokens)
{
  this.container = lcontainer;
  this.nmClient = nmClient;
  this.dag = sca.getContainer().getPlan().getLogicalPlan();
  this.tokens = tokens;
  this.sca = sca;
}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入依赖的package包/类
public void stop(NMClientAsync nmClient)
{
  if (stop == Long.MAX_VALUE) {
    nmClient.stopContainerAsync(container.getId(), container.getNodeId());
    LOG.info("Requested stop container {}", container.getId().toString());
    stop = System.currentTimeMillis();
  }
}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入依赖的package包/类
public YarnService(Config config, String applicationName, ApplicationId applicationId, FileSystem fs,
    EventBus eventBus, String containerJvmArgs) throws Exception {
  this.applicationName = applicationName;
  this.applicationId = applicationId;

  this.config = config;

  this.yarnConfiguration = new YarnConfiguration();
  this.amrmClientAsync = closer.register(
      AMRMClientAsync.createAMRMClientAsync(1000, new AMRMClientCallbackHandler()));
  this.amrmClientAsync.init(this.yarnConfiguration);
  this.nmClientAsync = closer.register(NMClientAsync.createNMClientAsync(new NMClientCallbackHandler()));
  this.nmClientAsync.init(this.yarnConfiguration);

  this.containerLaunchExecutor = Executors.newFixedThreadPool(10,
      ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("ContainerLaunchExecutor")));

  this.fs = fs;

  this.eventBus = eventBus;

  this.initialContainers = config.getInt(GobblinYarnConfigurationKeys.INITIAL_CONTAINERS_KEY);
  this.requestedContainerMemoryMbs = config.getInt(GobblinYarnConfigurationKeys.CONTAINER_MEMORY_MBS_KEY);
  this.requestedContainerCores = config.getInt(GobblinYarnConfigurationKeys.CONTAINER_CORES_KEY);

  this.containerJvmArgs = containerJvmArgs;

  this.tokens = getSecurityTokens();
}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入依赖的package包/类
public void initialize(AMRMClientAsync<AMRMClient.ContainerRequest> amRMClient
                      , NMClientAsync nmClient
                      , ServiceDiscoverer serviceDiscoverer
                      )
{
  this.nmClient = nmClient;
  this.amRMClient = amRMClient;
  this.serviceDiscoverer = serviceDiscoverer;
}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入依赖的package包/类
public YarnService(Config config, String applicationName, String applicationId, YarnConfiguration yarnConfiguration,
    FileSystem fs, EventBus eventBus) throws Exception {
  this.applicationName = applicationName;
  this.applicationId = applicationId;

  this.config = config;

  this.eventBus = eventBus;

  this.gobblinMetrics = config.getBoolean(ConfigurationKeys.METRICS_ENABLED_KEY) ?
      Optional.of(buildGobblinMetrics()) : Optional.<GobblinMetrics>absent();

  this.eventSubmitter = config.getBoolean(ConfigurationKeys.METRICS_ENABLED_KEY) ?
      Optional.of(buildEventSubmitter()) : Optional.<EventSubmitter>absent();

  this.yarnConfiguration = yarnConfiguration;
  this.fs = fs;

  this.amrmClientAsync = closer.register(
      AMRMClientAsync.createAMRMClientAsync(1000, new AMRMClientCallbackHandler()));
  this.amrmClientAsync.init(this.yarnConfiguration);
  this.nmClientAsync = closer.register(NMClientAsync.createNMClientAsync(new NMClientCallbackHandler()));
  this.nmClientAsync.init(this.yarnConfiguration);

  this.initialContainers = config.getInt(GobblinYarnConfigurationKeys.INITIAL_CONTAINERS_KEY);
  this.requestedContainerMemoryMbs = config.getInt(GobblinYarnConfigurationKeys.CONTAINER_MEMORY_MBS_KEY);
  this.requestedContainerCores = config.getInt(GobblinYarnConfigurationKeys.CONTAINER_CORES_KEY);
  this.containerHostAffinityEnabled = config.getBoolean(GobblinYarnConfigurationKeys.CONTAINER_HOST_AFFINITY_ENABLED);

  this.helixInstanceMaxRetries = config.getInt(GobblinYarnConfigurationKeys.HELIX_INSTANCE_MAX_RETRIES);

  this.containerJvmArgs = config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_JVM_ARGS_KEY) ?
      Optional.of(config.getString(GobblinYarnConfigurationKeys.CONTAINER_JVM_ARGS_KEY)) :
      Optional.<String>absent();

  this.containerLaunchExecutor = Executors.newFixedThreadPool(10,
      ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("ContainerLaunchExecutor")));

  this.tokens = getSecurityTokens();
}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入依赖的package包/类
@Override
protected void initialize() throws Exception {
  super.initialize();
  transitionTaskStateQueue = new SystemTaskQueue(this::handleException);

  // Initialize AM NoDependenceConfig
  conf.initializeNoDependenceConfig();

  // Start RMClient to Register AM ASAP in case AM expired by RM
  rmClient = AMRMClientAsync.createAMRMClientAsync(
      conf.getAmRmHeartbeatIntervalSec() * 1000,
      new RMClientCallbackHandler(this));
  rmClient.init(conf.getYarnConfig());
  rmClient.start();
  conf.initializeDependOnRMResponseConfig(registerToRM());

  // Start NMClient
  nmClient = NMClientAsync.createNMClientAsync(
      new NMClientCallbackHandler(this));
  nmClient.init(conf.getYarnConfig());
  nmClient.start();

  // Start YarnClient
  yarnClient = YarnClient.createYarnClient();
  yarnClient.init(conf.getYarnConfig());
  yarnClient.start();
  conf.initializeDependOnYarnClientConfig(yarnClient);

  // Initialize Launcher Store
  zkStore = new ZookeeperStore(conf.getZkConnectString(), conf.getZkRootDir());
  conf.initializeDependOnZKStoreConfig(zkStore);
  hdfsStore = new HdfsStore(conf.getLauncherConfig().getHdfsRootDir());
  hdfsStore.makeFrameworkRootDir(conf.getFrameworkName());
  hdfsStore.makeAMStoreRootDir(conf.getFrameworkName());

  // Initialize other components
  launcherClient = new LauncherClientInternal(
      conf.getLauncherConfig().getWebServerAddress(), 30, 10,
      LaunchClientType.APPLICATION_MASTER);

  gpuAllocationManager = new GpuAllocationManager(this);
  rmResyncHandler = new RMResyncHandler(this, conf);
}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入依赖的package包/类
NMClientAsync getNMClientAsync() {
  return nmClientAsync;
}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入依赖的package包/类
public NMClientAsyncImpl(CallbackHandler callbackHandler) {
  this(NMClientAsync.class.getName(), callbackHandler);
}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入依赖的package包/类
public NMClientAsyncImpl(AbstractCallbackHandler callbackHandler) {
  this(NMClientAsync.class.getName(), callbackHandler);
}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入依赖的package包/类
protected NMClientAsync createNMClientAsync() {
  launchListener = new NMCallback();
  nmClientAsync = NMClientAsync.createNMClientAsync(launchListener);
  nmClientAsync.init(context.getConf());
  return nmClientAsync;
}
 

import org.apache.hadoop.yarn.client.api.async.NMClientAsync; //导入依赖的package包/类
private void mockNMClientAsync() throws NoSuchFieldException, SecurityException, IllegalArgumentException, IllegalAccessException {
  nmClientAsync = mock(NMClientAsync.class);
  Field nmClientAsyncField = containerLauncher.getClass().getDeclaredField("nmClientAsync");
  nmClientAsyncField.setAccessible(true);
  nmClientAsyncField.set(containerLauncher, nmClientAsync);
}