Java DataNodeTestUtils.setHeartbeatsDisabledForTests方法代码示例

import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; //导入方法依赖的package包/类
/**
 * Stop the heartbeat of a datanode in the MiniDFSCluster
 * 
 * @param cluster
 *          The MiniDFSCluster
 * @param hostName
 *          The hostName of the datanode to be stopped
 * @return The DataNode whose heartbeat has been stopped
 */
private DataNode stopDataNodeHeartbeat(MiniDFSCluster cluster, String hostName) {
  for (DataNode dn : cluster.getDataNodes()) {
    if (dn.getDatanodeId().getHostName().equals(hostName)) {
      DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
      return dn;
    }
  }
  return null;
}
 

import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; //导入方法依赖的package包/类
@Before
public void setup() throws IOException {
  conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, INTERNAL_BLOCK_SIZE);
  conf.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_MAX_STREAMS_KEY, 0);
  SimulatedFSDataset.setFactory(conf);
  cluster = new MiniDFSCluster.Builder(conf).numDataNodes(
      DATA_BLK_NUM + PARITY_BLK_NUM).build();
  cluster.waitActive();
  for (DataNode dn : cluster.getDataNodes()) {
    DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
  }
  fs = cluster.getFileSystem();
  fs.mkdirs(dirPath);
  fs.getClient().setErasureCodingPolicy(dirPath.toString(), null);
}
 

import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; //导入方法依赖的package包/类
private void waitForHeartbeat(final DataNode dn, final DatanodeDescriptor dnd)
    throws Exception {
  final long lastUpdate = dnd.getLastUpdateMonotonic();
  Thread.sleep(1);
  DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, false);
  DataNodeTestUtils.triggerHeartbeat(dn);
  GenericTestUtils.waitFor(new Supplier<Boolean>() {
    @Override
    public Boolean get() {
      return lastUpdate != dnd.getLastUpdateMonotonic();
    }
  }, 10, 100000);
  DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
}
 

import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; //导入方法依赖的package包/类
@Test
public void testInvalidateBlock() throws IOException {
  final Path file = new Path("/invalidate");
  final int length = 10;
  final byte[] bytes = StripedFileTestUtil.generateBytes(length);
  DFSTestUtil.writeFile(fs, file, bytes);

  int dnIndex = findFirstDataNode(file, cellSize * dataBlocks);
  Assert.assertNotEquals(-1, dnIndex);
  LocatedStripedBlock slb = (LocatedStripedBlock)fs.getClient()
      .getLocatedBlocks(file.toString(), 0, cellSize * dataBlocks).get(0);
  final LocatedBlock[] blks = StripedBlockUtil.parseStripedBlockGroup(slb,
      cellSize, dataBlocks, parityBlocks);
  final Block b = blks[0].getBlock().getLocalBlock();

  DataNode dn = cluster.getDataNodes().get(dnIndex);
  // disable the heartbeat from DN so that the invalidated block record is kept
  // in NameNode until heartbeat expires and NN mark the dn as dead
  DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);

  try {
    // delete the file
    fs.delete(file, true);
    // check the block is added to invalidateBlocks
    final FSNamesystem fsn = cluster.getNamesystem();
    final BlockManager bm = fsn.getBlockManager();
    DatanodeDescriptor dnd = NameNodeAdapter.getDatanode(fsn, dn.getDatanodeId());
    Assert.assertTrue(bm.containsInvalidateBlock(
        blks[0].getLocations()[0], b) || dnd.containsInvalidateBlock(b));
  } finally {
    DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, false);
  }
}
 

import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; //导入方法依赖的package包/类
/**
 * Stop the heartbeat of a datanode in the MiniDFSCluster
 *
 * @param cluster
 *     The MiniDFSCluster
 * @param hostName
 *     The hostName of the datanode to be stopped
 * @return The DataNode whose heartbeat has been stopped
 */
private DataNode stopDataNodeHeartbeat(MiniDFSCluster cluster,
    String hostName) {
  for (DataNode dn : cluster.getDataNodes()) {
    if (dn.getDatanodeId().getHostName().equals(hostName)) {
      DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
      return dn;
    }
  }
  return null;
}
 

import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; //导入方法依赖的package包/类
/**
 * Test if BlockManager can correctly remove corresponding pending records
 * when a file is deleted
 * 
 * @throws Exception
 */
@Test
public void testPendingAndInvalidate() throws Exception {
  final Configuration CONF = new HdfsConfiguration();
  CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024);
  CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
      DFS_REPLICATION_INTERVAL);
  CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 
      DFS_REPLICATION_INTERVAL);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(
      DATANODE_COUNT).build();
  cluster.waitActive();
  
  FSNamesystem namesystem = cluster.getNamesystem();
  BlockManager bm = namesystem.getBlockManager();
  DistributedFileSystem fs = cluster.getFileSystem();
  try {
    // 1. create a file
    Path filePath = new Path("/tmp.txt");
    DFSTestUtil.createFile(fs, filePath, 1024, (short) 3, 0L);
    
    // 2. disable the heartbeats
    for (DataNode dn : cluster.getDataNodes()) {
      DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
    }
    
    // 3. mark a couple of blocks as corrupt
    LocatedBlock block = NameNodeAdapter.getBlockLocations(
        cluster.getNameNode(), filePath.toString(), 0, 1).get(0);
    cluster.getNamesystem().writeLock();
    try {
      bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0],
          "STORAGE_ID", "TEST");
      bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[1],
          "STORAGE_ID", "TEST");
    } finally {
      cluster.getNamesystem().writeUnlock();
    }
    BlockManagerTestUtil.computeAllPendingWork(bm);
    BlockManagerTestUtil.updateState(bm);
    assertEquals(bm.getPendingReplicationBlocksCount(), 1L);
    assertEquals(bm.pendingReplications.getNumReplicas(block.getBlock()
        .getLocalBlock()), 2);
    
    // 4. delete the file
    fs.delete(filePath, true);
    // retry at most 10 times, each time sleep for 1s. Note that 10s is much
    // less than the default pending record timeout (5~10min)
    int retries = 10; 
    long pendingNum = bm.getPendingReplicationBlocksCount();
    while (pendingNum != 0 && retries-- > 0) {
      Thread.sleep(1000);  // let NN do the deletion
      BlockManagerTestUtil.updateState(bm);
      pendingNum = bm.getPendingReplicationBlocksCount();
    }
    assertEquals(pendingNum, 0L);
  } finally {
    cluster.shutdown();
  }
}
 

import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; //导入方法依赖的package包/类
/** Corrupt a block and ensure metrics reflects it */
@Test
public void testCorruptBlock() throws Exception {
  // Create a file with single block with two replicas
  final Path file = getTestPath("testCorruptBlock");
  createFile(file, 100, (short)2);
  
  // Disable the heartbeats, so that no corrupted replica
  // can be fixed
  for (DataNode dn : cluster.getDataNodes()) {
    DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
  }
  
  // Corrupt first replica of the block
  LocatedBlock block = NameNodeAdapter.getBlockLocations(
      cluster.getNameNode(), file.toString(), 0, 1).get(0);
  cluster.getNamesystem().writeLock();
  try {
    bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0],
        "STORAGE_ID", "TEST");
  } finally {
    cluster.getNamesystem().writeUnlock();
  }
  BlockManagerTestUtil.getComputedDatanodeWork(bm);
  MetricsRecordBuilder rb = getMetrics(NS_METRICS);
  assertGauge("CorruptBlocks", 1L, rb);
  assertGauge("PendingReplicationBlocks", 1L, rb);
  
  fs.delete(file, true);
  // During the file deletion, both BlockManager#corruptReplicas and
  // BlockManager#pendingReplications will be updated, i.e., the records
  // for the blocks of the deleted file will be removed from both
  // corruptReplicas and pendingReplications. The corresponding
  // metrics (CorruptBlocks and PendingReplicationBlocks) will only be updated
  // when BlockManager#computeDatanodeWork is run where the
  // BlockManager#updateState is called. And in
  // BlockManager#computeDatanodeWork the metric ScheduledReplicationBlocks
  // will also be updated.
  rb = waitForDnMetricValue(NS_METRICS, "CorruptBlocks", 0L);
  assertGauge("PendingReplicationBlocks", 0L, rb);
  assertGauge("ScheduledReplicationBlocks", 0L, rb);
}
 

import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; //导入方法依赖的package包/类
/**
 * Test if BlockManager can correctly remove corresponding pending records
 * when a file is deleted
 * 
 * @throws Exception
 */
@Test
public void testPendingAndInvalidate() throws Exception {
  final Configuration CONF = new HdfsConfiguration();
  CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024);
  CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
      DFS_REPLICATION_INTERVAL);
  CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 
      DFS_REPLICATION_INTERVAL);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(
      DATANODE_COUNT).build();
  cluster.waitActive();
  
  FSNamesystem namesystem = cluster.getNamesystem();
  BlockManager bm = namesystem.getBlockManager();
  DistributedFileSystem fs = cluster.getFileSystem();
  try {
    // 1. create a file
    Path filePath = new Path("/tmp.txt");
    DFSTestUtil.createFile(fs, filePath, 1024, (short) 3, 0L);
    
    // 2. disable the heartbeats
    for (DataNode dn : cluster.getDataNodes()) {
      DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
    }
    
    // 3. mark a couple of blocks as corrupt
    LocatedBlock block = NameNodeAdapter.getBlockLocations(
        cluster.getNameNode(), filePath.toString(), 0, 1).get(0);
    cluster.getNamesystem().writeLock();
    try {
      bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0],
          "STORAGE_ID", "TEST");
      bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[1],
          "STORAGE_ID", "TEST");
    } finally {
      cluster.getNamesystem().writeUnlock();
    }
    BlockManagerTestUtil.computeAllPendingWork(bm);
    BlockManagerTestUtil.updateState(bm);
    assertEquals(bm.getPendingReplicationBlocksCount(), 1L);
    BlockInfo storedBlock = bm.getStoredBlock(block.getBlock().getLocalBlock());
    assertEquals(bm.pendingReplications.getNumReplicas(storedBlock), 2);

    // 4. delete the file
    fs.delete(filePath, true);
    // retry at most 10 times, each time sleep for 1s. Note that 10s is much
    // less than the default pending record timeout (5~10min)
    int retries = 10; 
    long pendingNum = bm.getPendingReplicationBlocksCount();
    while (pendingNum != 0 && retries-- > 0) {
      Thread.sleep(1000);  // let NN do the deletion
      BlockManagerTestUtil.updateState(bm);
      pendingNum = bm.getPendingReplicationBlocksCount();
    }
    assertEquals(pendingNum, 0L);
  } finally {
    cluster.shutdown();
  }
}
 

import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; //导入方法依赖的package包/类
/**
 * Test if BlockManager can correctly remove corresponding pending records
 * when a file is deleted
 * 
 * @throws Exception
 */
@Test
public void testPendingAndInvalidate() throws Exception {
  final Configuration CONF = new HdfsConfiguration();
  CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024);
  CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
      DFS_REPLICATION_INTERVAL);
  CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY, 
      DFS_REPLICATION_INTERVAL);
  MiniDFSCluster cluster = new MiniDFSCluster.Builder(CONF).numDataNodes(
      DATANODE_COUNT).build();
  cluster.waitActive();
  
  FSNamesystem namesystem = cluster.getNamesystem();
  BlockManager bm = namesystem.getBlockManager();
  DistributedFileSystem fs = cluster.getFileSystem();
  try {
    // 1. create a file
    Path filePath = new Path("/tmp.txt");
    DFSTestUtil.createFile(fs, filePath, 1024, (short) 3, 0L);
    
    // 2. disable the heartbeats
    for (DataNode dn : cluster.getDataNodes()) {
      DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
    }
    
    // 3. mark a couple of blocks as corrupt
    LocatedBlock block = NameNodeAdapter.getBlockLocations(
        cluster.getNameNode(), filePath.toString(), 0, 1).get(0);
    cluster.getNamesystem().writeLock();
    try {
      bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0],
          "TEST");
      bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[1],
          "TEST");
    } finally {
      cluster.getNamesystem().writeUnlock();
    }
    BlockManagerTestUtil.computeAllPendingWork(bm);
    BlockManagerTestUtil.updateState(bm);
    assertEquals(bm.getPendingReplicationBlocksCount(), 1L);
    assertEquals(bm.pendingReplications.getNumReplicas(block.getBlock()
        .getLocalBlock()), 2);
    
    // 4. delete the file
    fs.delete(filePath, true);
    // retry at most 10 times, each time sleep for 1s. Note that 10s is much
    // less than the default pending record timeout (5~10min)
    int retries = 10; 
    long pendingNum = bm.getPendingReplicationBlocksCount();
    while (pendingNum != 0 && retries-- > 0) {
      Thread.sleep(1000);  // let NN do the deletion
      BlockManagerTestUtil.updateState(bm);
      pendingNum = bm.getPendingReplicationBlocksCount();
    }
    assertEquals(pendingNum, 0L);
  } finally {
    cluster.shutdown();
  }
}
 

import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; //导入方法依赖的package包/类
/**
 * Test if BlockManager can correctly remove corresponding pending records
 * when a file is deleted
 *
 * @throws Exception
 */
@Test
public void testPendingAndInvalidate() throws Exception {
  final Configuration CONF = new HdfsConfiguration();
  CONF.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024);
  CONF.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
      DFS_REPLICATION_INTERVAL);
  CONF.setInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
      DFS_REPLICATION_INTERVAL);
  MiniDFSCluster cluster =
      new MiniDFSCluster.Builder(CONF).numDataNodes(DATANODE_COUNT).build();
  cluster.waitActive();
  
  FSNamesystem namesystem = cluster.getNamesystem();
  BlockManager bm = namesystem.getBlockManager();
  DistributedFileSystem fs = cluster.getFileSystem();
  try {
    // 1. create a file
    Path filePath = new Path("/tmp.txt");
    DFSTestUtil.createFile(fs, filePath, 1024, (short) 3, 0L);
    
    // 2. disable the heartbeats
    for (DataNode dn : cluster.getDataNodes()) {
      DataNodeTestUtils.setHeartbeatsDisabledForTests(dn, true);
    }
    
    // 3. mark a couple of blocks as corrupt
    LocatedBlock block = NameNodeAdapter
        .getBlockLocations(cluster.getNameNode(), filePath.toString(), 0, 1)
        .get(0);
    bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[0],
        "TEST");
    bm.findAndMarkBlockAsCorrupt(block.getBlock(), block.getLocations()[1],
        "TEST");
    BlockManagerTestUtil.computeAllPendingWork(bm);
    BlockManagerTestUtil.updateState(bm);
    assertEquals(bm.getPendingReplicationBlocksCount(), 1L);
    assertEquals(getNumReplicas(bm.pendingReplications,
        (BlockInfo) block.getBlock().getLocalBlock()), 2);
    
    // 4. delete the file
    fs.delete(filePath, true);
    // retry at most 10 times, each time sleep for 1s. Note that 10s is much
    // less than the default pending record timeout (5~10min)
    int retries = 10;
    long pendingNum = bm.getPendingReplicationBlocksCount();
    while (pendingNum != 0 && retries-- > 0) {
      Thread.sleep(1000);  // let NN do the deletion
      BlockManagerTestUtil.updateState(bm);
      pendingNum = bm.getPendingReplicationBlocksCount();
    }
    assertEquals(pendingNum, 0L);
  } finally {
    cluster.shutdown();
  }
}