Java NetworkTopology.getFirstHalf方法代码示例

import org.apache.hadoop.net.NetworkTopology; //导入方法依赖的package包/类
/**
 * {@inheritDoc}
 */
@Override
protected void chooseRemoteRack(int numOfReplicas,
    DatanodeDescriptor localMachine, Set<Node> excludedNodes,
    long blocksize, int maxReplicasPerRack, List<DatanodeStorageInfo> results,
    boolean avoidStaleNodes, EnumMap<StorageType, Integer> storageTypes)
    throws NotEnoughReplicasException {
  int oldNumOfReplicas = results.size();

  final String rackLocation = NetworkTopology.getFirstHalf(
      localMachine.getNetworkLocation());
  try {
    // randomly choose from remote racks
    chooseRandom(numOfReplicas, "~" + rackLocation, excludedNodes, blocksize,
        maxReplicasPerRack, results, avoidStaleNodes, storageTypes);
  } catch (NotEnoughReplicasException e) {
    // fall back to the local rack
    chooseRandom(numOfReplicas - (results.size() - oldNumOfReplicas),
        rackLocation, excludedNodes, blocksize,
        maxReplicasPerRack, results, avoidStaleNodes, storageTypes);
  }
}
 

import org.apache.hadoop.net.NetworkTopology; //导入方法依赖的package包/类
@Override
protected void chooseRemoteRack(int numOfReplicas,
    DatanodeDescriptor localMachine, Set<Node> excludedNodes,
    long blocksize, int maxReplicasPerRack, List<DatanodeStorageInfo> results,
    boolean avoidStaleNodes, EnumMap<StorageType, Integer> storageTypes)
    throws NotEnoughReplicasException {
  int oldNumOfReplicas = results.size();

  final String rackLocation = NetworkTopology.getFirstHalf(
      localMachine.getNetworkLocation());
  try {
    // randomly choose from remote racks
    chooseRandom(numOfReplicas, "~" + rackLocation, excludedNodes, blocksize,
        maxReplicasPerRack, results, avoidStaleNodes, storageTypes);
  } catch (NotEnoughReplicasException e) {
    // fall back to the local rack
    chooseRandom(numOfReplicas - (results.size() - oldNumOfReplicas),
        rackLocation, excludedNodes, blocksize,
        maxReplicasPerRack, results, avoidStaleNodes, storageTypes);
  }
}
 

import org.apache.hadoop.net.NetworkTopology; //导入方法依赖的package包/类
@Override
protected DatanodeStorageInfo chooseLocalRack(Node localMachine,
    Set<Node> excludedNodes, long blocksize, int maxNodesPerRack,
    List<DatanodeStorageInfo> results, boolean avoidStaleNodes,
    EnumMap<StorageType, Integer> storageTypes) throws
    NotEnoughReplicasException {
  // no local machine, so choose a random machine
  if (localMachine == null) {
    return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
        maxNodesPerRack, results, avoidStaleNodes, storageTypes);
  }

  // choose one from the local rack, but off-nodegroup
  try {
    final String scope = NetworkTopology.getFirstHalf(localMachine.getNetworkLocation());
    return chooseRandom(scope, excludedNodes, blocksize, maxNodesPerRack,
        results, avoidStaleNodes, storageTypes);
  } catch (NotEnoughReplicasException e1) {
    // find the second replica
    final DatanodeDescriptor newLocal = secondNode(localMachine, results);
    if (newLocal != null) {
      try {
        return chooseRandom(
            clusterMap.getRack(newLocal.getNetworkLocation()), excludedNodes,
            blocksize, maxNodesPerRack, results, avoidStaleNodes,
            storageTypes);
      } catch(NotEnoughReplicasException e2) {
        //otherwise randomly choose one from the network
        return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
            maxNodesPerRack, results, avoidStaleNodes, storageTypes);
      }
    } else {
      //otherwise randomly choose one from the network
      return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
          maxNodesPerRack, results, avoidStaleNodes, storageTypes);
    }
  }
}
 

import org.apache.hadoop.net.NetworkTopology; //导入方法依赖的package包/类
@Override
protected String getRack(final DatanodeInfo cur) {
  String nodeGroupString = cur.getNetworkLocation();
  return NetworkTopology.getFirstHalf(nodeGroupString);
}
 

import org.apache.hadoop.net.NetworkTopology; //导入方法依赖的package包/类
/**
 * Create a cluster with even distribution, and a new empty node is added to
 * the cluster, then test rack locality for balancer policy. 
 */
@Test(timeout=60000)
public void testBalancerWithRackLocality() throws Exception {
  Configuration conf = createConf();
  long[] capacities = new long[]{CAPACITY, CAPACITY};
  String[] racks = new String[]{RACK0, RACK1};
  String[] nodeGroups = new String[]{NODEGROUP0, NODEGROUP1};
  
  int numOfDatanodes = capacities.length;
  assertEquals(numOfDatanodes, racks.length);
  MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf)
                              .numDataNodes(capacities.length)
                              .racks(racks)
                              .simulatedCapacities(capacities);
  MiniDFSClusterWithNodeGroup.setNodeGroups(nodeGroups);
  cluster = new MiniDFSClusterWithNodeGroup(builder);
  try {
    cluster.waitActive();
    client = NameNodeProxies.createProxy(conf, 
        cluster.getFileSystem(0).getUri(),
        ClientProtocol.class).getProxy();

    long totalCapacity = TestBalancer.sum(capacities);

    // fill up the cluster to be 30% full
    long totalUsedSpace = totalCapacity * 3 / 10;
    TestBalancer.createFile(cluster, filePath, totalUsedSpace / numOfDatanodes,
        (short) numOfDatanodes, 0);
    
    long newCapacity = CAPACITY;
    String newRack = RACK1;
    String newNodeGroup = NODEGROUP2;
    // start up an empty node with the same capacity and on the same rack
    cluster.startDataNodes(conf, 1, true, null, new String[]{newRack},
        new long[] {newCapacity}, new String[]{newNodeGroup});

    totalCapacity += newCapacity;

    // run balancer and validate results
    runBalancerCanFinish(conf, totalUsedSpace, totalCapacity);
    
    DatanodeInfo[] datanodeReport = 
            client.getDatanodeReport(DatanodeReportType.ALL);
    
    Map<String, Integer> rackToUsedCapacity = new HashMap<String, Integer>();
    for (DatanodeInfo datanode: datanodeReport) {
      String rack = NetworkTopology.getFirstHalf(datanode.getNetworkLocation());
      int usedCapacity = (int) datanode.getDfsUsed();
       
      if (rackToUsedCapacity.get(rack) != null) {
        rackToUsedCapacity.put(rack, usedCapacity + rackToUsedCapacity.get(rack));
      } else {
        rackToUsedCapacity.put(rack, usedCapacity);
      }
    }
    assertEquals(rackToUsedCapacity.size(), 2);
    assertEquals(rackToUsedCapacity.get(RACK0), rackToUsedCapacity.get(RACK1));
    
  } finally {
    cluster.shutdown();
  }
}