Java ResourceBlacklistRequest.newInstance方法代码示例

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入方法依赖的package包/类
@Override
public ResourceBlacklistRequest getBlacklistUpdates() {
  ResourceBlacklistRequest ret;
  List<String> blacklist = new ArrayList<>(blacklistNodes);
  final int currentBlacklistSize = blacklist.size();
  final double failureThreshold = this.blacklistDisableFailureThreshold *
      numberOfNodeManagerHosts;
  if (currentBlacklistSize < failureThreshold) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("blacklist size " + currentBlacklistSize + " is less than " +
          "failure threshold ratio " + blacklistDisableFailureThreshold +
          " out of total usable nodes " + numberOfNodeManagerHosts);
    }
    ret = ResourceBlacklistRequest.newInstance(blacklist, EMPTY_LIST);
  } else {
    LOG.warn("Ignoring Blacklists, blacklist size " + currentBlacklistSize
        + " is more than failure threshold ratio "
        + blacklistDisableFailureThreshold + " out of total usable nodes "
        + numberOfNodeManagerHosts);
    // TODO: After the threshold hits, we will keep sending a long list
    // every time a new AM is to be scheduled.
    ret = ResourceBlacklistRequest.newInstance(EMPTY_LIST, blacklist);
  }
  return ret;
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入方法依赖的package包/类
protected AllocateResponse makeRemoteRequest() throws YarnException,
    IOException {
  ResourceBlacklistRequest blacklistRequest =
      ResourceBlacklistRequest.newInstance(new ArrayList<String>(blacklistAdditions),
          new ArrayList<String>(blacklistRemovals));
  AllocateRequest allocateRequest =
      AllocateRequest.newInstance(lastResponseID,
        super.getApplicationProgress(), new ArrayList<ResourceRequest>(ask),
        new ArrayList<ContainerId>(release), blacklistRequest);
  AllocateResponse allocateResponse = scheduler.allocate(allocateRequest);
  lastResponseID = allocateResponse.getResponseId();
  availableResources = allocateResponse.getAvailableResources();
  lastClusterNmCount = clusterNmCount;
  clusterNmCount = allocateResponse.getNumClusterNodes();

  if (ask.size() > 0 || release.size() > 0) {
    LOG.info("getResources() for " + applicationId + ":" + " ask="
        + ask.size() + " release= " + release.size() + " newContainers="
        + allocateResponse.getAllocatedContainers().size()
        + " finishedContainers="
        + allocateResponse.getCompletedContainersStatuses().size()
        + " resourcelimit=" + availableResources + " knownNMs="
        + clusterNmCount);
  }

  ask.clear();
  release.clear();

  if (blacklistAdditions.size() > 0 || blacklistRemovals.size() > 0) {
    LOG.info("Update the blacklist for " + applicationId +
        ": blacklistAdditions=" + blacklistAdditions.size() +
        " blacklistRemovals=" +  blacklistRemovals.size());
  }
  blacklistAdditions.clear();
  blacklistRemovals.clear();
  return allocateResponse;
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入方法依赖的package包/类
private AllocateRequest createAllocateRequest(List<NodeReport> listNode) {
  // The test needs AMRMClient to create a real allocate request
  AMRMClientImpl<ContainerRequest> amClient =
      new AMRMClientImpl<ContainerRequest>();

  Resource capability = Resource.newInstance(1024, 2);
  Priority priority = Priority.newInstance(1);
  List<NodeReport> nodeReports = listNode;
  String node = nodeReports.get(0).getNodeId().getHost();
  String[] nodes = new String[] { node };

  ContainerRequest storedContainer1 =
      new ContainerRequest(capability, nodes, null, priority);
  amClient.addContainerRequest(storedContainer1);
  amClient.addContainerRequest(storedContainer1);

  List<ResourceRequest> resourceAsk = new ArrayList<ResourceRequest>();
  for (ResourceRequest rr : amClient.ask) {
    resourceAsk.add(rr);
  }

  ResourceBlacklistRequest resourceBlacklistRequest = ResourceBlacklistRequest
      .newInstance(new ArrayList<String>(), new ArrayList<String>());

  int responseId = 1;

  return AllocateRequest.newInstance(responseId, 0, resourceAsk,
      new ArrayList<ContainerId>(), resourceBlacklistRequest);
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入方法依赖的package包/类
@Test
public void testValidateResourceBlacklistRequest() throws Exception {

  MyContainerManager containerManager = new MyContainerManager();
  final MockRMWithAMS rm =
      new MockRMWithAMS(new YarnConfiguration(), containerManager);
  rm.start();

  MockNM nm1 = rm.registerNode("localhost:1234", 5120);

  Map<ApplicationAccessType, String> acls =
      new HashMap<ApplicationAccessType, String>(2);
  acls.put(ApplicationAccessType.VIEW_APP, "*");
  RMApp app = rm.submitApp(1024, "appname", "appuser", acls);

  nm1.nodeHeartbeat(true);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
  waitForLaunchedState(attempt);

  // Create a client to the RM.
  final Configuration conf = rm.getConfig();
  final YarnRPC rpc = YarnRPC.create(conf);

  UserGroupInformation currentUser = 
      UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
  Credentials credentials = containerManager.getContainerCredentials();
  final InetSocketAddress rmBindAddress =
      rm.getApplicationMasterService().getBindAddress();
  Token<? extends TokenIdentifier> amRMToken =
      MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress,
        credentials.getAllTokens());
  currentUser.addToken(amRMToken);
  ApplicationMasterProtocol client =
      currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
          return (ApplicationMasterProtocol) rpc.getProxy(
            ApplicationMasterProtocol.class, rmBindAddress, conf);
        }
      });

  RegisterApplicationMasterRequest request = Records
      .newRecord(RegisterApplicationMasterRequest.class);
  client.registerApplicationMaster(request);

  ResourceBlacklistRequest blacklistRequest =
      ResourceBlacklistRequest.newInstance(
          Collections.singletonList(ResourceRequest.ANY), null);

  AllocateRequest allocateRequest =
      AllocateRequest.newInstance(0, 0.0f, null, null, blacklistRequest);
  boolean error = false;
  try {
    client.allocate(allocateRequest);
  } catch (InvalidResourceBlacklistRequestException e) {
    error = true;
  }

  rm.stop();
  
  Assert.assertTrue(
      "Didn't not catch InvalidResourceBlacklistRequestException", error);
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入方法依赖的package包/类
protected AllocateResponse makeRemoteRequest() throws YarnException,
    IOException {
  applyRequestLimits();
  ResourceBlacklistRequest blacklistRequest =
      ResourceBlacklistRequest.newInstance(new ArrayList<String>(blacklistAdditions),
          new ArrayList<String>(blacklistRemovals));
  AllocateRequest allocateRequest =
      AllocateRequest.newInstance(lastResponseID,
        super.getApplicationProgress(), new ArrayList<ResourceRequest>(ask),
        new ArrayList<ContainerId>(release), blacklistRequest);
  AllocateResponse allocateResponse = scheduler.allocate(allocateRequest);
  lastResponseID = allocateResponse.getResponseId();
  availableResources = allocateResponse.getAvailableResources();
  lastClusterNmCount = clusterNmCount;
  clusterNmCount = allocateResponse.getNumClusterNodes();
  int numCompletedContainers =
      allocateResponse.getCompletedContainersStatuses().size();

  if (ask.size() > 0 || release.size() > 0) {
    LOG.info("getResources() for " + applicationId + ":" + " ask="
        + ask.size() + " release= " + release.size() + " newContainers="
        + allocateResponse.getAllocatedContainers().size()
        + " finishedContainers=" + numCompletedContainers
        + " resourcelimit=" + availableResources + " knownNMs="
        + clusterNmCount);
  }

  ask.clear();
  release.clear();

  if (numCompletedContainers > 0) {
    // re-send limited requests when a container completes to trigger asking
    // for more containers
    requestLimitsToUpdate.addAll(requestLimits.keySet());
  }

  if (blacklistAdditions.size() > 0 || blacklistRemovals.size() > 0) {
    LOG.info("Update the blacklist for " + applicationId +
        ": blacklistAdditions=" + blacklistAdditions.size() +
        " blacklistRemovals=" +  blacklistRemovals.size());
  }
  blacklistAdditions.clear();
  blacklistRemovals.clear();
  return allocateResponse;
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入方法依赖的package包/类
protected AllocateResponse makeRemoteRequest() throws YarnException,
    IOException {
  ResourceBlacklistRequest blacklistRequest =
      ResourceBlacklistRequest.newInstance(new ArrayList<String>(blacklistAdditions),
          new ArrayList<String>(blacklistRemovals));
  AllocateRequest allocateRequest =
      AllocateRequest.newInstance(lastResponseID,
        super.getApplicationProgress(), new ArrayList<ResourceRequest>(ask),
        new ArrayList<ContainerId>(release), blacklistRequest);
  AllocateResponse allocateResponse = scheduler.allocate(allocateRequest);
  lastResponseID = allocateResponse.getResponseId();
  availableResources = allocateResponse.getAvailableResources();
  lastClusterNmCount = clusterNmCount;
  clusterNmCount = allocateResponse.getNumClusterNodes();
  
  if (ask.size() > 0 || release.size() > 0 || allocateResponse.getAllocatedContainers().size() > 0) {
  
    LOG.info("one time for request containers ask size:"+ask.size());	
    
    for(ResourceRequest request:ask){
  	 
    	   LOG.info("container on"+request.getResourceName()+"relax locality"+request.getRelaxLocality()+"priority:"+request.getPriority().toString()+"number of containers"+request.getNumContainers());
    }
    
    LOG.info("getResources() for " + applicationId + ":" + " ask="
        + ask.size() + " release= " + release.size() + " newContainers="
        + allocateResponse.getAllocatedContainers().size()
        + " finishedContainers="
        + allocateResponse.getCompletedContainersStatuses().size()
        + " resourcelimit=" + availableResources + " knownNMs="
        + clusterNmCount);
  }

  ask.clear();
  release.clear();
  
  LOG.info("after release, ask size"+ask.size());

  if (blacklistAdditions.size() > 0 || blacklistRemovals.size() > 0) {
    LOG.info("Update the blacklist for " + applicationId +
        ": blacklistAdditions=" + blacklistAdditions.size() +
        " blacklistRemovals=" +  blacklistRemovals.size());
  }
  blacklistAdditions.clear();
  blacklistRemovals.clear();
  return allocateResponse;
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入方法依赖的package包/类
@Test
public void testValidateResourceBlacklistRequest() throws Exception {

  final Configuration conf = new YarnConfiguration();
  RMStorageFactory.setConfiguration(conf);
  YarnAPIStorageFactory.setConfiguration(conf);
  DBUtility.InitializeDB();

  MyContainerManager containerManager = new MyContainerManager();
  final MockRMWithAMS rm =
      new MockRMWithAMS(conf, containerManager);
  rm.start();

  MockNM nm1 = rm.registerNode("localhost:1234", 5120);

  Map<ApplicationAccessType, String> acls =
      new HashMap<ApplicationAccessType, String>(2);
  acls.put(ApplicationAccessType.VIEW_APP, "*");
  RMApp app = rm.submitApp(1024, "appname", "appuser", acls);

  nm1.nodeHeartbeat(true);

  RMAppAttempt attempt = app.getCurrentAppAttempt();
  ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
  waitForLaunchedState(attempt);

  // Create a client to the RM.
  final YarnRPC rpc = YarnRPC.create(conf);

  UserGroupInformation currentUser = 
      UserGroupInformation.createRemoteUser(applicationAttemptId.toString(), false);
  Credentials credentials = containerManager.getContainerCredentials();
  final InetSocketAddress rmBindAddress =
      rm.getApplicationMasterService().getBindAddress();
  Token<? extends TokenIdentifier> amRMToken =
      MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress,
        credentials.getAllTokens());
  currentUser.addToken(amRMToken);
  ApplicationMasterProtocol client =
      currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
        @Override
        public ApplicationMasterProtocol run() {
          return (ApplicationMasterProtocol) rpc.getProxy(
            ApplicationMasterProtocol.class, rmBindAddress, conf);
        }
      });

  RegisterApplicationMasterRequest request = Records
      .newRecord(RegisterApplicationMasterRequest.class);
  client.registerApplicationMaster(request);

  ResourceBlacklistRequest blacklistRequest =
      ResourceBlacklistRequest.newInstance(
          Collections.singletonList(ResourceRequest.ANY), null);

  AllocateRequest allocateRequest =
      AllocateRequest.newInstance(0, 0.0f, null, null, blacklistRequest);
  boolean error = false;
  try {
    client.allocate(allocateRequest);
  } catch (InvalidResourceBlacklistRequestException e) {
    error = true;
  }

  rm.stop();
  
  Assert.assertTrue(
      "Didn't not catch InvalidResourceBlacklistRequestException", error);
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入方法依赖的package包/类
protected AllocateResponse makeRemoteRequest() throws IOException {
  ResourceBlacklistRequest blacklistRequest =
      ResourceBlacklistRequest.newInstance(new ArrayList<String>(blacklistAdditions),
          new ArrayList<String>(blacklistRemovals));
  AllocateRequest allocateRequest =
      AllocateRequest.newInstance(lastResponseID,
        super.getApplicationProgress(), new ArrayList<ResourceRequest>(ask),
        new ArrayList<ContainerId>(release), blacklistRequest);
  AllocateResponse allocateResponse;
  try {
    allocateResponse = scheduler.allocate(allocateRequest);
  } catch (YarnException e) {
    throw new IOException(e);
  }
  lastResponseID = allocateResponse.getResponseId();
  availableResources = allocateResponse.getAvailableResources();
  lastClusterNmCount = clusterNmCount;
  clusterNmCount = allocateResponse.getNumClusterNodes();

  if (ask.size() > 0 || release.size() > 0) {
    LOG.info("getResources() for " + applicationId + ":" + " ask="
        + ask.size() + " release= " + release.size() + " newContainers="
        + allocateResponse.getAllocatedContainers().size()
        + " finishedContainers="
        + allocateResponse.getCompletedContainersStatuses().size()
        + " resourcelimit=" + availableResources + " knownNMs="
        + clusterNmCount);
  }

  ask.clear();
  release.clear();

  if (blacklistAdditions.size() > 0 || blacklistRemovals.size() > 0) {
    LOG.info("Update the blacklist for " + applicationId +
        ": blacklistAdditions=" + blacklistAdditions.size() +
        " blacklistRemovals=" +  blacklistRemovals.size());
  }
  blacklistAdditions.clear();
  blacklistRemovals.clear();
  return allocateResponse;
}