Java ResourceBlacklistRequest类代码示例

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入依赖的package包/类
@Public
@Stable
public static AllocateRequest newInstance(int responseID, float appProgress,
    List<ResourceRequest> resourceAsk,
    List<ContainerId> containersToBeReleased,
    ResourceBlacklistRequest resourceBlacklistRequest,
    List<ContainerResourceIncreaseRequest> increaseRequests) {
  AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
  allocateRequest.setResponseId(responseID);
  allocateRequest.setProgress(appProgress);
  allocateRequest.setAskList(resourceAsk);
  allocateRequest.setReleaseList(containersToBeReleased);
  allocateRequest.setResourceBlacklistRequest(resourceBlacklistRequest);
  allocateRequest.setIncreaseRequests(increaseRequests);
  return allocateRequest;
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入依赖的package包/类
@Public
@Stable
public static AllocateRequest newInstance(int responseID, float appProgress,
    List<ResourceRequest> resourceAsk,
    List<ContainerId> containersToBeReleased,
    ResourceBlacklistRequest resourceBlacklistRequest,
    List<ContainerResourceChangeRequest> increaseRequests,
    List<ContainerResourceChangeRequest> decreaseRequests) {
  AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
  allocateRequest.setResponseId(responseID);
  allocateRequest.setProgress(appProgress);
  allocateRequest.setAskList(resourceAsk);
  allocateRequest.setReleaseList(containersToBeReleased);
  allocateRequest.setResourceBlacklistRequest(resourceBlacklistRequest);
  allocateRequest.setIncreaseRequests(increaseRequests);
  allocateRequest.setDecreaseRequests(decreaseRequests);
  return allocateRequest;
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入依赖的package包/类
@Override
public ResourceBlacklistRequest getBlacklistUpdates() {
  ResourceBlacklistRequest ret;
  List<String> blacklist = new ArrayList<>(blacklistNodes);
  final int currentBlacklistSize = blacklist.size();
  final double failureThreshold = this.blacklistDisableFailureThreshold *
      numberOfNodeManagerHosts;
  if (currentBlacklistSize < failureThreshold) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("blacklist size " + currentBlacklistSize + " is less than " +
          "failure threshold ratio " + blacklistDisableFailureThreshold +
          " out of total usable nodes " + numberOfNodeManagerHosts);
    }
    ret = ResourceBlacklistRequest.newInstance(blacklist, EMPTY_LIST);
  } else {
    LOG.warn("Ignoring Blacklists, blacklist size " + currentBlacklistSize
        + " is more than failure threshold ratio "
        + blacklistDisableFailureThreshold + " out of total usable nodes "
        + numberOfNodeManagerHosts);
    // TODO: After the threshold hits, we will keep sending a long list
    // every time a new AM is to be scheduled.
    ret = ResourceBlacklistRequest.newInstance(EMPTY_LIST, blacklist);
  }
  return ret;
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入依赖的package包/类
@Test
public void testSimpleBlacklistBelowFailureThreshold() {
  final int numberOfNodeManagerHosts = 3;
  final double blacklistDisableFailureThreshold = 0.8;
  BlacklistManager manager = new SimpleBlacklistManager(
      numberOfNodeManagerHosts, blacklistDisableFailureThreshold);
  String anyNode = "foo";
  String anyNode2 = "bar";
  manager.addNode(anyNode);
  manager.addNode(anyNode2);
  ResourceBlacklistRequest blacklist = manager
      .getBlacklistUpdates();

  List<String> blacklistAdditions = blacklist.getBlacklistAdditions();
  Collections.sort(blacklistAdditions);
  List<String> blacklistRemovals = blacklist.getBlacklistRemovals();
  String[] expectedBlacklistAdditions = new String[]{anyNode2, anyNode};
  Assert.assertArrayEquals(
      "Blacklist additions was not as expected",
      expectedBlacklistAdditions,
      blacklistAdditions.toArray());
  Assert.assertTrue(
      "Blacklist removals should be empty but was " +
          blacklistRemovals,
      blacklistRemovals.isEmpty());
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入依赖的package包/类
@Test
public void testDisabledBlacklist() {
  BlacklistManager disabled = new DisabledBlacklistManager();
  String anyNode = "foo";
  disabled.addNode(anyNode);
  ResourceBlacklistRequest blacklist = disabled
      .getBlacklistUpdates();

  List<String> blacklistAdditions = blacklist.getBlacklistAdditions();
  List<String> blacklistRemovals = blacklist.getBlacklistRemovals();
  Assert.assertTrue(
      "Blacklist additions should be empty but was " +
          blacklistAdditions,
      blacklistAdditions.isEmpty());
  Assert.assertTrue(
      "Blacklist removals should be empty but was " +
          blacklistRemovals,
      blacklistRemovals.isEmpty());
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入依赖的package包/类
@Test(timeout = 15000)
public void testAllocateOnHA() throws YarnException, IOException, InterruptedException {
  AllocateRequest request = AllocateRequest.newInstance(0, 50f,
      new ArrayList<ResourceRequest>(),
      new ArrayList<ContainerId>(),
      ResourceBlacklistRequest.newInstance(new ArrayList<String>(),
          new ArrayList<String>()));
  int nbTry = 0;
  AllocateResponse response = null;
  while (nbTry < 10) {
    try {
      response = amClient.allocate(request);
      break;
    } catch (IOException ex) {
      if (!(ex instanceof SecretManager.InvalidToken)) {
        throw ex;
      }
    }
    Thread.sleep(200);
    nbTry++;
  }
  Assert.assertEquals(response, this.cluster.createFakeAllocateResponse());
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入依赖的package包/类
/**
 * Use {@link AllocateRequest#newInstance(int, float, List, List,
 * ResourceBlacklistRequest, List)} instead
 * @param responseID responseId
 * @param appProgress appProgress
 * @param resourceAsk resourceAsk
 * @param containersToBeReleased containersToBeReleased
 * @param resourceBlacklistRequest resourceBlacklistRequest
 * @param increaseRequests increaseRequests
 * @return AllocateRequest
 */
@Deprecated
public static AllocateRequest newInstance(int responseID, float appProgress,
    List<ResourceRequest> resourceAsk,
    List<ContainerId> containersToBeReleased,
    ResourceBlacklistRequest resourceBlacklistRequest,
    List<ContainerResourceIncreaseRequest> increaseRequests) {
  AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
  allocateRequest.setResponseId(responseID);
  allocateRequest.setProgress(appProgress);
  allocateRequest.setAskList(resourceAsk);
  allocateRequest.setReleaseList(containersToBeReleased);
  allocateRequest.setResourceBlacklistRequest(resourceBlacklistRequest);
  allocateRequest.setIncreaseRequests(increaseRequests);
  return allocateRequest;
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入依赖的package包/类
@Override
public ResourceBlacklistRequest getResourceBlacklistRequest() {
  AllocateRequestProtoOrBuilder p = viaProto ? proto : builder;
  if (this.blacklistRequest != null) {
    return this.blacklistRequest;
  }
  if (!p.hasBlacklistRequest()) {
    return null;
  }
  this.blacklistRequest = convertFromProtoFormat(p.getBlacklistRequest());
  return this.blacklistRequest;
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入依赖的package包/类
@Override
public void setResourceBlacklistRequest(ResourceBlacklistRequest blacklistRequest) {
  maybeInitBuilder();
  if (blacklistRequest == null) {
    builder.clearBlacklistRequest();
  }
  this.blacklistRequest = blacklistRequest;
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入依赖的package包/类
public static void validateBlacklistRequest(
    ResourceBlacklistRequest blacklistRequest)
    throws InvalidResourceBlacklistRequestException {
  if (blacklistRequest != null) {
    List<String> plus = blacklistRequest.getBlacklistAdditions();
    if (plus != null && plus.contains(ResourceRequest.ANY)) {
      throw new InvalidResourceBlacklistRequestException(
          "Cannot add " + ResourceRequest.ANY + " to the blacklist!");
    }
  }
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入依赖的package包/类
@Test(timeout = 15000)
public void testAllocateOnHA() throws YarnException, IOException {
  AllocateRequest request = AllocateRequest.newInstance(0, 50f,
      new ArrayList<ResourceRequest>(),
      new ArrayList<ContainerId>(),
      ResourceBlacklistRequest.newInstance(new ArrayList<String>(),
          new ArrayList<String>()));
  AllocateResponse response = amClient.allocate(request);
  Assert.assertEquals(response, this.cluster.createFakeAllocateResponse());
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入依赖的package包/类
protected AllocateResponse makeRemoteRequest() throws YarnException,
    IOException {
  ResourceBlacklistRequest blacklistRequest =
      ResourceBlacklistRequest.newInstance(new ArrayList<String>(blacklistAdditions),
          new ArrayList<String>(blacklistRemovals));
  AllocateRequest allocateRequest =
      AllocateRequest.newInstance(lastResponseID,
        super.getApplicationProgress(), new ArrayList<ResourceRequest>(ask),
        new ArrayList<ContainerId>(release), blacklistRequest);
  AllocateResponse allocateResponse = scheduler.allocate(allocateRequest);
  lastResponseID = allocateResponse.getResponseId();
  availableResources = allocateResponse.getAvailableResources();
  lastClusterNmCount = clusterNmCount;
  clusterNmCount = allocateResponse.getNumClusterNodes();

  if (ask.size() > 0 || release.size() > 0) {
    LOG.info("getResources() for " + applicationId + ":" + " ask="
        + ask.size() + " release= " + release.size() + " newContainers="
        + allocateResponse.getAllocatedContainers().size()
        + " finishedContainers="
        + allocateResponse.getCompletedContainersStatuses().size()
        + " resourcelimit=" + availableResources + " knownNMs="
        + clusterNmCount);
  }

  ask.clear();
  release.clear();

  if (blacklistAdditions.size() > 0 || blacklistRemovals.size() > 0) {
    LOG.info("Update the blacklist for " + applicationId +
        ": blacklistAdditions=" + blacklistAdditions.size() +
        " blacklistRemovals=" +  blacklistRemovals.size());
  }
  blacklistAdditions.clear();
  blacklistRemovals.clear();
  return allocateResponse;
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入依赖的package包/类
public static void validateBlacklistRequest(ResourceBlacklistRequest blacklistRequest) 
throws InvalidResourceBlacklistRequestException {
  if (blacklistRequest != null) {
    List<String> plus = blacklistRequest.getBlacklistAdditions();
    if (plus != null && plus.contains(ResourceRequest.ANY)) {
      throw new InvalidResourceBlacklistRequestException(
          "Cannot add " + ResourceRequest.ANY + " to the blacklist!");
    }
  }
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入依赖的package包/类
@Public
@Stable
public static AllocateRequest newInstance(int responseID, float appProgress,
    List<ResourceRequest> resourceAsk,
    List<ContainerId> containersToBeReleased,
    ResourceBlacklistRequest resourceBlacklistRequest) {
  AllocateRequest allocateRequest = Records.newRecord(AllocateRequest.class);
  allocateRequest.setResponseId(responseID);
  allocateRequest.setProgress(appProgress);
  allocateRequest.setAskList(resourceAsk);
  allocateRequest.setReleaseList(containersToBeReleased);
  allocateRequest.setResourceBlacklistRequest(resourceBlacklistRequest);
  return allocateRequest;
}
 

import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest; //导入依赖的package包/类
private AllocateRequest createAllocateRequest(List<NodeReport> listNode) {
  // The test needs AMRMClient to create a real allocate request
  AMRMClientImpl<ContainerRequest> amClient =
      new AMRMClientImpl<ContainerRequest>();

  Resource capability = Resource.newInstance(1024, 2);
  Priority priority = Priority.newInstance(1);
  List<NodeReport> nodeReports = listNode;
  String node = nodeReports.get(0).getNodeId().getHost();
  String[] nodes = new String[] { node };

  ContainerRequest storedContainer1 =
      new ContainerRequest(capability, nodes, null, priority);
  amClient.addContainerRequest(storedContainer1);
  amClient.addContainerRequest(storedContainer1);

  List<ResourceRequest> resourceAsk = new ArrayList<ResourceRequest>();
  for (ResourceRequest rr : amClient.ask) {
    resourceAsk.add(rr);
  }

  ResourceBlacklistRequest resourceBlacklistRequest = ResourceBlacklistRequest
      .newInstance(new ArrayList<String>(), new ArrayList<String>());

  int responseId = 1;

  return AllocateRequest.newInstance(responseId, 0, resourceAsk,
      new ArrayList<ContainerId>(), resourceBlacklistRequest);
}