Java ResourceWeights类代码示例

import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; //导入依赖的package包/类
public AllocationConfiguration(Configuration conf) {
  minQueueResources = new HashMap<String, Resource>();
  maxQueueResources = new HashMap<String, Resource>();
  queueWeights = new HashMap<String, ResourceWeights>();
  queueMaxApps = new HashMap<String, Integer>();
  userMaxApps = new HashMap<String, Integer>();
  queueMaxAMShares = new HashMap<String, Float>();
  userMaxAppsDefault = Integer.MAX_VALUE;
  queueMaxAppsDefault = Integer.MAX_VALUE;
  queueMaxAMShareDefault = 0.5f;
  queueAcls = new HashMap<String, Map<QueueACL, AccessControlList>>();
  minSharePreemptionTimeouts = new HashMap<String, Long>();
  fairSharePreemptionTimeouts = new HashMap<String, Long>();
  fairSharePreemptionThresholds = new HashMap<String, Float>();
  schedulingPolicies = new HashMap<String, SchedulingPolicy>();
  defaultSchedulingPolicy = SchedulingPolicy.DEFAULT_POLICY;
  reservableQueues = new HashSet<>();
  configuredQueues = new HashMap<FSQueueType, Set<String>>();
  for (FSQueueType queueType : FSQueueType.values()) {
    configuredQueues.put(queueType, new HashSet<String>());
  }
  placementPolicy = QueuePlacementPolicy.fromConfiguration(conf,
      configuredQueues);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; //导入依赖的package包/类
@Test
public void testCalculateShares() {
  Resource used = Resources.createResource(10, 5, 8);
  Resource capacity = Resources.createResource(100, 10, 20);
  ResourceType[] resourceOrder = new ResourceType[3];
  ResourceWeights shares = new ResourceWeights();
  ((DominantResourceFairnessPolicy.DominantResourceFairnessComparator)
      createComparator(100, 10, 20)).calculateShares(
      used, capacity, shares, resourceOrder, ResourceWeights.NEUTRAL);
  
  assertEquals(.1, shares.getWeight(ResourceType.MEMORY), .00001);
  assertEquals(.5, shares.getWeight(ResourceType.CPU), .00001);
  assertEquals(.4, shares.getWeight(ResourceType.GPU), .00001);
  assertEquals(ResourceType.CPU, resourceOrder[0]);
  assertEquals(ResourceType.GPU, resourceOrder[1]);
  assertEquals(ResourceType.MEMORY, resourceOrder[2]);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; //导入依赖的package包/类
public AllocationConfiguration(Configuration conf) {
  minQueueResources = new HashMap<String, Resource>();
  maxQueueResources = new HashMap<String, Resource>();
  queueWeights = new HashMap<String, ResourceWeights>();
  queueMaxApps = new HashMap<String, Integer>();
  userMaxApps = new HashMap<String, Integer>();
  queueMaxAMShares = new HashMap<String, Float>();
  userMaxAppsDefault = Integer.MAX_VALUE;
  queueMaxAppsDefault = Integer.MAX_VALUE;
  queueMaxResourcesDefault = Resources.unbounded();
  queueMaxAMShareDefault = 0.5f;
  queueAcls = new HashMap<String, Map<QueueACL, AccessControlList>>();
  minSharePreemptionTimeouts = new HashMap<String, Long>();
  fairSharePreemptionTimeouts = new HashMap<String, Long>();
  fairSharePreemptionThresholds = new HashMap<String, Float>();
  schedulingPolicies = new HashMap<String, SchedulingPolicy>();
  defaultSchedulingPolicy = SchedulingPolicy.DEFAULT_POLICY;
  reservableQueues = new HashSet<>();
  configuredQueues = new HashMap<FSQueueType, Set<String>>();
  for (FSQueueType queueType : FSQueueType.values()) {
    configuredQueues.put(queueType, new HashSet<String>());
  }
  placementPolicy = QueuePlacementPolicy.fromConfiguration(conf,
      configuredQueues);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; //导入依赖的package包/类
/**
 * Calculates and orders a resource's share of a pool in terms of two vectors.
 * The shares vector contains, for each resource, the fraction of the pool that
 * it takes up.  The resourceOrder vector contains an ordering of resources
 * by largest share.  So if resource=<10 MB, 5 CPU>, and pool=<100 MB, 10 CPU>,
 * shares will be [.1, .5] and resourceOrder will be [CPU, MEMORY].
 */
void calculateShares(Resource resource, Resource pool,
    ResourceWeights shares, ResourceType[] resourceOrder, ResourceWeights weights) {
  shares.setWeight(MEMORY, (float)resource.getMemory() /
      (pool.getMemory() * weights.getWeight(MEMORY)));
  shares.setWeight(CPU, (float)resource.getVirtualCores() /
      (pool.getVirtualCores() * weights.getWeight(CPU)));
  // sort order vector by resource share
  if (resourceOrder != null) {
    if (shares.getWeight(MEMORY) > shares.getWeight(CPU)) {
      resourceOrder[0] = MEMORY;
      resourceOrder[1] = CPU;
    } else  {
      resourceOrder[0] = CPU;
      resourceOrder[1] = MEMORY;
    }
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; //导入依赖的package包/类
@Test
public void testCalculateShares() {
  Resource used = Resources.createResource(10, 5);
  Resource capacity = Resources.createResource(100, 10);
  ResourceType[] resourceOrder = new ResourceType[2];
  ResourceWeights shares = new ResourceWeights();
  DominantResourceFairnessPolicy.DominantResourceFairnessComparator comparator =
      new DominantResourceFairnessPolicy.DominantResourceFairnessComparator();
  comparator.calculateShares(used, capacity, shares, resourceOrder,
      ResourceWeights.NEUTRAL);
  
  assertEquals(.1, shares.getWeight(ResourceType.MEMORY), .00001);
  assertEquals(.5, shares.getWeight(ResourceType.CPU), .00001);
  assertEquals(ResourceType.CPU, resourceOrder[0]);
  assertEquals(ResourceType.MEMORY, resourceOrder[1]);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; //导入依赖的package包/类
public AllocationConfiguration(Configuration conf) {
  minQueueResources = new HashMap<String, Resource>();
  maxQueueResources = new HashMap<String, Resource>();
  queueWeights = new HashMap<String, ResourceWeights>();
  queueMaxApps = new HashMap<String, Integer>();
  userMaxApps = new HashMap<String, Integer>();
  queueMaxAMShares = new HashMap<String, Float>();
  userMaxAppsDefault = Integer.MAX_VALUE;
  queueMaxAppsDefault = Integer.MAX_VALUE;
  queueMaxAMShareDefault = 0.5f;
  queueAcls = new HashMap<String, Map<QueueACL, AccessControlList>>();
  minSharePreemptionTimeouts = new HashMap<String, Long>();
  fairSharePreemptionTimeouts = new HashMap<String, Long>();
  fairSharePreemptionThresholds = new HashMap<String, Float>();
  schedulingPolicies = new HashMap<String, SchedulingPolicy>();
  defaultSchedulingPolicy = SchedulingPolicy.DEFAULT_POLICY;
  configuredQueues = new HashMap<FSQueueType, Set<String>>();
  for (FSQueueType queueType : FSQueueType.values()) {
    configuredQueues.put(queueType, new HashSet<String>());
  }
  placementPolicy = QueuePlacementPolicy.fromConfiguration(conf,
      configuredQueues);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; //导入依赖的package包/类
public QueueManagerInfo(Map<String, Resource> minQueueResources, 
    Map<String, Resource> maxQueueResources, 
    Map<String, Integer> queueMaxApps, Map<String, Integer> userMaxApps,
    Map<String, ResourceWeights> queueWeights, int userMaxAppsDefault,
    int queueMaxAppsDefault, SchedulingPolicy defaultSchedulingPolicy, 
    Map<String, Long> minSharePreemptionTimeouts, 
    Map<String, Map<QueueACL, AccessControlList>> queueAcls,
    long fairSharePreemptionTimeout, long defaultMinSharePreemptionTimeout) {
  this.minQueueResources = minQueueResources;
  this.maxQueueResources = maxQueueResources;
  this.queueMaxApps = queueMaxApps;
  this.userMaxApps = userMaxApps;
  this.queueWeights = queueWeights;
  this.userMaxAppsDefault = userMaxAppsDefault;
  this.queueMaxAppsDefault = queueMaxAppsDefault;
  this.defaultSchedulingPolicy = defaultSchedulingPolicy;
  this.minSharePreemptionTimeouts = minSharePreemptionTimeouts;
  this.queueAcls = queueAcls;
  this.fairSharePreemptionTimeout = fairSharePreemptionTimeout;
  this.defaultMinSharePreemptionTimeout = defaultMinSharePreemptionTimeout;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; //导入依赖的package包/类
public synchronized ResourceWeights getAppWeight(AppSchedulable app) {
  if (!app.getRunnable()) {
    // Job won't launch tasks, but don't return 0 to avoid division errors
    return ResourceWeights.NEUTRAL;
  } else {
    double weight = 1.0;
    if (sizeBasedWeight) {
      // Set weight based on current memory demand
      weight = Math.log1p(app.getDemand().getMemory()) / Math.log(2);
    }
    weight *= app.getPriority().getPriority();
    if (weightAdjuster != null) {
      // Run weight through the user-supplied weightAdjuster
      weight = weightAdjuster.adjustWeight(app, weight);
    }
    return new ResourceWeights((float)weight);
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; //导入依赖的package包/类
public AllocationConfiguration(Configuration conf) {
  minQueueResources = new HashMap<String, Resource>();
  maxQueueResources = new HashMap<String, Resource>();
  queueWeights = new HashMap<String, ResourceWeights>();
  queueMaxApps = new HashMap<String, Integer>();
  userMaxApps = new HashMap<String, Integer>();
  queueMaxAMShares = new HashMap<String, Float>();
  userMaxAppsDefault = Integer.MAX_VALUE;
  queueMaxAppsDefault = Integer.MAX_VALUE;
  queueMaxResourcesDefault = Resources.unbounded();
  queueMaxAMShareDefault = 0.5f;
  queueAcls = new HashMap<String, Map<QueueACL, AccessControlList>>();
  resAcls = new HashMap<String, Map<ReservationACL, AccessControlList>>();
  minSharePreemptionTimeouts = new HashMap<String, Long>();
  fairSharePreemptionTimeouts = new HashMap<String, Long>();
  fairSharePreemptionThresholds = new HashMap<String, Float>();
  schedulingPolicies = new HashMap<String, SchedulingPolicy>();
  defaultSchedulingPolicy = SchedulingPolicy.DEFAULT_POLICY;
  reservableQueues = new HashSet<>();
  configuredQueues = new HashMap<FSQueueType, Set<String>>();
  for (FSQueueType queueType : FSQueueType.values()) {
    configuredQueues.put(queueType, new HashSet<String>());
  }
  placementPolicy = QueuePlacementPolicy.fromConfiguration(conf,
      configuredQueues);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; //导入依赖的package包/类
/**
 * Calculates and orders a resource's share of a pool in terms of two vectors.
 * The shares vector contains, for each resource, the fraction of the pool that
 * it takes up.  The resourceOrder vector contains an ordering of resources
 * by largest share.  So if resource=<10 MB, 5 CPU>, and pool=<100 MB, 10 CPU>,
 * shares will be [.1, .5] and resourceOrder will be [CPU, MEMORY].
 */
void calculateShares(Resource resource, Resource pool,
    ResourceWeights shares, ResourceType[] resourceOrder, ResourceWeights weights) {
  shares.setWeight(MEMORY, (float)resource.getMemorySize() /
      (pool.getMemorySize() * weights.getWeight(MEMORY)));
  shares.setWeight(CPU, (float)resource.getVirtualCores() /
      (pool.getVirtualCores() * weights.getWeight(CPU)));
  // sort order vector by resource share
  if (resourceOrder != null) {
    if (shares.getWeight(MEMORY) > shares.getWeight(CPU)) {
      resourceOrder[0] = MEMORY;
      resourceOrder[1] = CPU;
      resourceOrder[2] = GPU;
    } else  {
      resourceOrder[0] = CPU;
      resourceOrder[1] = MEMORY;
      resourceOrder[2] = GPU;
    }
  }
}
 

import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; //导入依赖的package包/类
@Test
public void testCalculateShares() {
  Resource used = Resources.createResource(10, 5);
  Resource capacity = Resources.createResource(100, 10);
  ResourceType[] resourceOrder = new ResourceType[3];
  ResourceWeights shares = new ResourceWeights();
  DominantResourceFairnessPolicy.DominantResourceFairnessComparator comparator =
      new DominantResourceFairnessPolicy.DominantResourceFairnessComparator();
  comparator.calculateShares(used, capacity, shares, resourceOrder,
      ResourceWeights.NEUTRAL);
  
  assertEquals(.1, shares.getWeight(ResourceType.MEMORY), .00001);
  assertEquals(.5, shares.getWeight(ResourceType.CPU), .00001);
  assertEquals(ResourceType.CPU, resourceOrder[0]);
  assertEquals(ResourceType.MEMORY, resourceOrder[1]);
}
 

import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; //导入依赖的package包/类
public AllocationConfiguration(Map<String, Resource> minQueueResources, 
    Map<String, Resource> maxQueueResources, 
    Map<String, Integer> queueMaxApps, Map<String, Integer> userMaxApps,
    Map<String, ResourceWeights> queueWeights, int userMaxAppsDefault,
    int queueMaxAppsDefault, Map<String, SchedulingPolicy> schedulingPolicies,
    SchedulingPolicy defaultSchedulingPolicy,
    Map<String, Long> minSharePreemptionTimeouts, 
    Map<String, Map<QueueACL, AccessControlList>> queueAcls,
    long fairSharePreemptionTimeout, long defaultMinSharePreemptionTimeout,
    QueuePlacementPolicy placementPolicy, Set<String> queueNames) {
  this.minQueueResources = minQueueResources;
  this.maxQueueResources = maxQueueResources;
  this.queueMaxApps = queueMaxApps;
  this.userMaxApps = userMaxApps;
  this.queueWeights = queueWeights;
  this.userMaxAppsDefault = userMaxAppsDefault;
  this.queueMaxAppsDefault = queueMaxAppsDefault;
  this.defaultSchedulingPolicy = defaultSchedulingPolicy;
  this.schedulingPolicies = schedulingPolicies;
  this.minSharePreemptionTimeouts = minSharePreemptionTimeouts;
  this.queueAcls = queueAcls;
  this.fairSharePreemptionTimeout = fairSharePreemptionTimeout;
  this.defaultMinSharePreemptionTimeout = defaultMinSharePreemptionTimeout;
  this.placementPolicy = placementPolicy;
  this.queueNames = queueNames;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; //导入依赖的package包/类
public synchronized ResourceWeights getAppWeight(FSAppAttempt app) {
  double weight = 1.0;
  if (sizeBasedWeight) {
    // Set weight based on current memory demand
    weight = Math.log1p(app.getDemand().getMemory()) / Math.log(2);
  }
  weight *= app.getPriority().getPriority();
  if (weightAdjuster != null) {
    // Run weight through the user-supplied weightAdjuster
    weight = weightAdjuster.adjustWeight(app, weight);
  }
  ResourceWeights resourceWeights = app.getResourceWeights();
  resourceWeights.setWeight((float)weight);
  return resourceWeights;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; //导入依赖的package包/类
private int compareShares(ResourceWeights shares1, ResourceWeights shares2,
    ResourceType[] resourceOrder1, ResourceType[] resourceOrder2) {
  for (int i = 0; i < resourceOrder1.length; i++) {
    int ret = (int)Math.signum(shares1.getWeight(resourceOrder1[i])
        - shares2.getWeight(resourceOrder2[i]));
    if (ret != 0) {
      return ret;
    }
  }
  return 0;
}
 

import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceWeights; //导入依赖的package包/类
public FSAppAttempt(FairScheduler scheduler,
    ApplicationAttemptId applicationAttemptId, String user, FSLeafQueue queue,
    ActiveUsersManager activeUsersManager, RMContext rmContext) {
  super(applicationAttemptId, user, queue, activeUsersManager, rmContext);

  this.scheduler = scheduler;
  this.startTime = scheduler.getClock().getTime();
  this.priority = Priority.newInstance(1);
  this.resourceWeights = new ResourceWeights();
}