Java DFSUtil.shuffle方法代码示例

import org.apache.hadoop.hdfs.DFSUtil; //导入方法依赖的package包/类
/** Get live datanode storage reports and then build the network topology. */
public List<DatanodeStorageReport> init() throws IOException {
  final DatanodeStorageReport[] reports = nnc.getLiveDatanodeStorageReport();
  final List<DatanodeStorageReport> trimmed = new ArrayList<DatanodeStorageReport>(); 
  // create network topology and classify utilization collections:
  // over-utilized, above-average, below-average and under-utilized.
  for (DatanodeStorageReport r : DFSUtil.shuffle(reports)) {
    final DatanodeInfo datanode = r.getDatanodeInfo();
    if (shouldIgnore(datanode)) {
      continue;
    }
    trimmed.add(r);
    cluster.add(datanode);
  }
  return trimmed;
}
 

import org.apache.hadoop.hdfs.DFSUtil; //导入方法依赖的package包/类
/**
 * Choose <i>localMachine</i> as the target.
 * if <i>localMachine</i> is not available, 
 * choose a node on the same rack
 * @return the chosen storage
 */
protected DatanodeStorageInfo chooseLocalStorage(Node localMachine,
    Set<Node> excludedNodes, long blocksize, int maxNodesPerRack,
    List<DatanodeStorageInfo> results, boolean avoidStaleNodes,
    EnumMap<StorageType, Integer> storageTypes, boolean fallbackToLocalRack)
    throws NotEnoughReplicasException {
  // if no local machine, randomly choose one node
  if (localMachine == null) {
    return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
        maxNodesPerRack, results, avoidStaleNodes, storageTypes);
  }
  if (preferLocalNode && localMachine instanceof DatanodeDescriptor) {
    DatanodeDescriptor localDatanode = (DatanodeDescriptor) localMachine;
    // otherwise try local machine first
    if (excludedNodes.add(localMachine)) { // was not in the excluded list
      for (Iterator<Map.Entry<StorageType, Integer>> iter = storageTypes
          .entrySet().iterator(); iter.hasNext(); ) {
        Map.Entry<StorageType, Integer> entry = iter.next();
        for (DatanodeStorageInfo localStorage : DFSUtil.shuffle(
            localDatanode.getStorageInfos())) {
          StorageType type = entry.getKey();
          if (addIfIsGoodTarget(localStorage, excludedNodes, blocksize,
              maxNodesPerRack, false, results, avoidStaleNodes, type) >= 0) {
            int num = entry.getValue();
            if (num == 1) {
              iter.remove();
            } else {
              entry.setValue(num - 1);
            }
            return localStorage;
          }
        }
      }
    } 
  }

  if (!fallbackToLocalRack) {
    return null;
  }
  // try a node on local rack
  return chooseLocalRack(localMachine, excludedNodes, blocksize,
      maxNodesPerRack, results, avoidStaleNodes, storageTypes);
}
 

import org.apache.hadoop.hdfs.DFSUtil; //导入方法依赖的package包/类
/** choose local node of localMachine as the target.
 * if localMachine is not available, choose a node on the same nodegroup or 
 * rack instead.
 * @return the chosen node
 */
@Override
protected DatanodeStorageInfo chooseLocalStorage(Node localMachine,
    Set<Node> excludedNodes, long blocksize, int maxNodesPerRack,
    List<DatanodeStorageInfo> results, boolean avoidStaleNodes,
    EnumMap<StorageType, Integer> storageTypes, boolean fallbackToLocalRack)
    throws NotEnoughReplicasException {
  // if no local machine, randomly choose one node
  if (localMachine == null)
    return chooseRandom(NodeBase.ROOT, excludedNodes, 
        blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes);

  // otherwise try local machine first
  if (localMachine instanceof DatanodeDescriptor) {
    DatanodeDescriptor localDataNode = (DatanodeDescriptor)localMachine;
    if (excludedNodes.add(localMachine)) { // was not in the excluded list
      for (Iterator<Map.Entry<StorageType, Integer>> iter = storageTypes
          .entrySet().iterator(); iter.hasNext(); ) {
        Map.Entry<StorageType, Integer> entry = iter.next();
        for (DatanodeStorageInfo localStorage : DFSUtil.shuffle(
            localDataNode.getStorageInfos())) {
          StorageType type = entry.getKey();
          if (addIfIsGoodTarget(localStorage, excludedNodes, blocksize,
              maxNodesPerRack, false, results, avoidStaleNodes, type) >= 0) {
            int num = entry.getValue();
            if (num == 1) {
              iter.remove();
            } else {
              entry.setValue(num - 1);
            }
            return localStorage;
          }
        }
      }
    }
  }

  // try a node on local node group
  DatanodeStorageInfo chosenStorage = chooseLocalNodeGroup(
      (NetworkTopologyWithNodeGroup)clusterMap, localMachine, excludedNodes, 
      blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes);
  if (chosenStorage != null) {
    return chosenStorage;
  }

  if (!fallbackToLocalRack) {
    return null;
  }
  // try a node on local rack
  return chooseLocalRack(localMachine, excludedNodes, 
      blocksize, maxNodesPerRack, results, avoidStaleNodes, storageTypes);
}