Java NumberReplicas.decommissionedReplicas方法代码示例

import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.NumberReplicas; //导入方法依赖的package包/类
public void testUnderReplicationWithDecommissionDataNode() throws Exception {
  final Configuration conf = new Configuration();
  final short REPLICATION_FACTOR = (short)1;
  File f = new File(HOST_FILE_PATH);
  if (f.exists()) {
    f.delete();
  }
  f.createNewFile();
  conf.set("dfs.hosts.exclude", HOST_FILE_PATH);
  LOG.info("Start the cluster");
  final MiniDFSCluster cluster = 
    new MiniDFSCluster(conf, REPLICATION_FACTOR, true, null);
  try {
    final FSNamesystem namesystem = cluster.getNameNode().namesystem;
    final FileSystem fs = cluster.getFileSystem();
    DatanodeDescriptor[] datanodes = (DatanodeDescriptor[])
          namesystem.heartbeats.toArray(
              new DatanodeDescriptor[REPLICATION_FACTOR]);
    assertEquals(1, datanodes.length);
    // populate the cluster with a one block file
    final Path FILE_PATH = new Path("/testfile2");
    DFSTestUtil.createFile(fs, FILE_PATH, 1L, REPLICATION_FACTOR, 1L);
    DFSTestUtil.waitReplication(fs, FILE_PATH, REPLICATION_FACTOR);
    Block block = DFSTestUtil.getFirstBlock(fs, FILE_PATH);

    // shutdown the datanode
    DataNodeProperties dnprop = shutdownDataNode(cluster, datanodes[0]);
    assertEquals(1, namesystem.getMissingBlocksCount()); // one missing block
    assertEquals(0, namesystem.getNonCorruptUnderReplicatedBlocks());

    // Make the only datanode to be decommissioned
    LOG.info("Decommission the datanode " + dnprop);
    addToExcludeFile(namesystem.getConf(), datanodes);
    namesystem.refreshNodes(namesystem.getConf());      
    
    // bring up the datanode
    cluster.restartDataNode(dnprop);

    // Wait for block report
    LOG.info("wait for its block report to come in");
    NumberReplicas num;
    long startTime = System.currentTimeMillis();
    do {
     namesystem.readLock();
     try {
       num = namesystem.countNodes(block);
     } finally {
       namesystem.readUnlock();
     }
     Thread.sleep(1000);
     LOG.info("live: " + num.liveReplicas() 
         + "Decom: " + num.decommissionedReplicas());
    } while (num.decommissionedReplicas() != 1 &&
        System.currentTimeMillis() - startTime < 30000);
    assertEquals("Decommissioning Replicas doesn't reach 1", 
        1, num.decommissionedReplicas());
    assertEquals(1, namesystem.getNonCorruptUnderReplicatedBlocks());
    assertEquals(0, namesystem.getMissingBlocksCount());
  } finally {
    cluster.shutdown();
  }
}
 

import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.NumberReplicas; //导入方法依赖的package包/类
void metaSave(PrintWriter out) {
  //
  // Dump contents of neededReplication
  //
  synchronized (neededReplications) {
    out.println("Metasave: Blocks waiting for replication: " + 
                neededReplications.size());
    for (Block block : neededReplications) {
      List<DatanodeDescriptor> containingNodes =
                                        new ArrayList<DatanodeDescriptor>();
      NumberReplicas numReplicas = new NumberReplicas();
      // source node returned is not used
      chooseSourceDatanode(block, containingNodes, numReplicas);
      int usableReplicas = numReplicas.liveReplicas() +
                           numReplicas.decommissionedReplicas();
     
      if (block instanceof BlockInfo) {
        String fileName = ((BlockInfo)block).getINode().getFullPathName();
        out.print(fileName + ": ");
      }
      // l: == live:, d: == decommissioned c: == corrupt e: == excess
      out.print(block + ((usableReplicas > 0)? "" : " MISSING") + 
                " (replicas:" +
                " l: " + numReplicas.liveReplicas() +
                " d: " + numReplicas.decommissionedReplicas() +
                " c: " + numReplicas.corruptReplicas() +
                " e: " + numReplicas.excessReplicas() + ") "); 

      Collection<DatanodeDescriptor> corruptNodes = 
                                    corruptReplicas.getNodes(block);
      
      for (Iterator<DatanodeDescriptor> jt = blocksMap.nodeIterator(block);
           jt.hasNext();) {
        DatanodeDescriptor node = jt.next();
        String state = "";
        if (corruptNodes != null && corruptNodes.contains(node)) {
          state = "(corrupt)";
        } else if (node.isDecommissioned() || 
            node.isDecommissionInProgress()) {
          state = "(decommissioned)";
        }          
        out.print(" " + node + state + " : ");
      }
      out.println("");
    }
  }

  //
  // Dump blocks from pendingReplication
  //
  pendingReplications.metaSave(out);

  //
  // Dump blocks that are waiting to be deleted
  //
  dumpRecentInvalidateSets(out);
}
 

import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.NumberReplicas; //导入方法依赖的package包/类
/**
 * Return true if there are any blocks on this node that have not
 * yet reached their replication factor. Otherwise returns false.
 */
boolean isReplicationInProgress(DatanodeDescriptor srcNode) {
  boolean status = false;
  int underReplicatedBlocks = 0;
  int decommissionOnlyReplicas = 0;
  int underReplicatedInOpenFiles = 0;
  final Iterator<? extends Block> it = srcNode.getBlockIterator();
  while(it.hasNext()) {
    final Block block = it.next();
    INode fileINode = blocksMap.getINode(block);

    if (fileINode != null) {
      NumberReplicas num = countNodes(block);
      int curReplicas = num.liveReplicas();
      int curExpectedReplicas = getReplication(block);
      if (isNeededReplication(block, curExpectedReplicas, curReplicas)) {
        if (curExpectedReplicas > curReplicas) {
          //Log info about one block for this node which needs replication
          if (!status) {
            status = true;
            logBlockReplicationInfo(block, srcNode, num);
          }
          underReplicatedBlocks++;
          if ((curReplicas == 0) && (num.decommissionedReplicas() > 0)) {
            decommissionOnlyReplicas++;
          }
          if (fileINode.isUnderConstruction()) {
            underReplicatedInOpenFiles++;
          }
        }
        if (!neededReplications.contains(block) &&
          pendingReplications.getNumReplicas(block) == 0) {
          //
          // These blocks have been reported from the datanode
          // after the startDecommission method has been executed. These
          // blocks were in flight when the decommissioning was started.
          //
          neededReplications.add(block,
                                 curReplicas,
                                 num.decommissionedReplicas(),
                                 curExpectedReplicas);
        }
      }
    }
  }
  srcNode.decommissioningStatus.set(underReplicatedBlocks,
      decommissionOnlyReplicas, 
      underReplicatedInOpenFiles);
  return status;
}
 

import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.NumberReplicas; //导入方法依赖的package包/类
public void testUnderReplicationWithDecommissionDataNode() throws Exception {
  final Configuration conf = new Configuration();
  final short REPLICATION_FACTOR = (short)1;
  File f = new File(HOST_FILE_PATH);
  if (f.exists()) {
    f.delete();
  }
  conf.set("dfs.hosts.exclude", HOST_FILE_PATH);
  LOG.info("Start the cluster");
  final MiniDFSCluster cluster = 
    new MiniDFSCluster(conf, REPLICATION_FACTOR, true, null);
  try {
    final FSNamesystem namesystem = cluster.getNameNode().namesystem;
    final FileSystem fs = cluster.getFileSystem();
    DatanodeDescriptor[] datanodes = (DatanodeDescriptor[])
          namesystem.heartbeats.toArray(
              new DatanodeDescriptor[REPLICATION_FACTOR]);
    assertEquals(1, datanodes.length);
    // populate the cluster with a one block file
    final Path FILE_PATH = new Path("/testfile2");
    DFSTestUtil.createFile(fs, FILE_PATH, 1L, REPLICATION_FACTOR, 1L);
    DFSTestUtil.waitReplication(fs, FILE_PATH, REPLICATION_FACTOR);
    Block block = DFSTestUtil.getFirstBlock(fs, FILE_PATH);

    // shutdown the datanode
    DataNodeProperties dnprop = shutdownDataNode(cluster, datanodes[0]);
    assertEquals(1, namesystem.getMissingBlocksCount()); // one missing block
    assertEquals(0, namesystem.getNonCorruptUnderReplicatedBlocks());

    // Make the only datanode to be decommissioned
    LOG.info("Decommission the datanode " + dnprop);
    addToExcludeFile(namesystem.getConf(), datanodes);
    namesystem.refreshNodes(namesystem.getConf());      
    
    // bring up the datanode
    cluster.restartDataNode(dnprop);

    // Wait for block report
    LOG.info("wait for its block report to come in");
    NumberReplicas num;
    long startTime = System.currentTimeMillis();
    do {
     namesystem.readLock();
     try {
       num = namesystem.countNodes(block);
     } finally {
       namesystem.readUnlock();
     }
     Thread.sleep(1000);
     LOG.info("live: " + num.liveReplicas() 
         + "Decom: " + num.decommissionedReplicas());
    } while (num.decommissionedReplicas() != 1 &&
        System.currentTimeMillis() - startTime < 30000);
    assertEquals("Decommissioning Replicas doesn't reach 1", 
        1, num.decommissionedReplicas());
    assertEquals(1, namesystem.getNonCorruptUnderReplicatedBlocks());
    assertEquals(0, namesystem.getMissingBlocksCount());
  } finally {
    cluster.shutdown();
  }
}