Java DataNodeTestUtils.spyOnBposToNN方法代码示例

import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; //导入方法依赖的package包/类
/**
 * Another regression test for HDFS-2742. This tests the following sequence:
 * - DN does a block report while file is open. This BR contains
 *   the block in RBW state.
 * - The block report is delayed in reaching the standby.
 * - The file is closed.
 * - The standby processes the OP_ADD and OP_CLOSE operations before
 *   the RBW block report arrives.
 * - The standby should not mark the block as corrupt.
 */
@Test
public void testRBWReportArrivesAfterEdits() throws Exception {
  final CountDownLatch brFinished = new CountDownLatch(1);
  DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG) {
    @Override
    protected Object passThrough(InvocationOnMock invocation)
        throws Throwable {
      try {
        return super.passThrough(invocation);
      } finally {
        // inform the test that our block report went through.
        brFinished.countDown();
      }
    }
  };

  FSDataOutputStream out = fs.create(TEST_FILE_PATH);
  try {
    AppendTestUtil.write(out, 0, 10);
    out.hflush();

    DataNode dn = cluster.getDataNodes().get(0);
    DatanodeProtocolClientSideTranslatorPB spy =
      DataNodeTestUtils.spyOnBposToNN(dn, nn2);
    
    Mockito.doAnswer(delayer)
      .when(spy).blockReport(
        Mockito.<DatanodeRegistration>anyObject(),
        Mockito.anyString(),
        Mockito.<StorageBlockReport[]>anyObject(),
        Mockito.<BlockReportContext>anyObject());
    dn.scheduleAllBlockReport(0);
    delayer.waitForCall();
    
  } finally {
    IOUtils.closeStream(out);
  }

  cluster.transitionToStandby(0);
  cluster.transitionToActive(1);
  
  delayer.proceed();
  brFinished.await();
  
  // Verify that no replicas are marked corrupt, and that the
  // file is readable from the failed-over standby.
  BlockManagerTestUtil.updateState(nn1.getNamesystem().getBlockManager());
  BlockManagerTestUtil.updateState(nn2.getNamesystem().getBlockManager());
  assertEquals(0, nn1.getNamesystem().getCorruptReplicaBlocks());
  assertEquals(0, nn2.getNamesystem().getCorruptReplicaBlocks());
  
  DFSTestUtil.readFile(fs, TEST_FILE_PATH);
}
 

import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; //导入方法依赖的package包/类
/**
 * This test makes sure that, when a file is closed before all
 * of the datanodes in the pipeline have reported their replicas,
 * the NameNode doesn't consider the block under-replicated too
 * aggressively. It is a regression test for HDFS-1172.
 */
@Test(timeout=60000)
public void testNoExtraReplicationWhenBlockReceivedIsLate()
    throws Exception {
  LOG.info("Test block replication when blockReceived is late" );
  final short numDataNodes = 3;
  final short replication = 3;
  final Configuration conf = new Configuration();
      conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024);
  final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
      .numDataNodes(numDataNodes).build();
  final String testFile = "/replication-test-file";
  final Path testPath = new Path(testFile);
  final BlockManager bm =
      cluster.getNameNode().getNamesystem().getBlockManager();

  try {
    cluster.waitActive();

    // Artificially delay IBR from 1 DataNode.
    // this ensures that the client's completeFile() RPC will get to the
    // NN before some of the replicas are reported.
    NameNode nn = cluster.getNameNode();
    DataNode dn = cluster.getDataNodes().get(0);
    DatanodeProtocolClientSideTranslatorPB spy =
        DataNodeTestUtils.spyOnBposToNN(dn, nn);
    DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG);
    Mockito.doAnswer(delayer).when(spy).blockReceivedAndDeleted(
        Mockito.<DatanodeRegistration>anyObject(),
        Mockito.anyString(),
        Mockito.<StorageReceivedDeletedBlocks[]>anyObject());

    FileSystem fs = cluster.getFileSystem();
    // Create and close a small file with two blocks
    DFSTestUtil.createFile(fs, testPath, 1500, replication, 0);

    // schedule replication via BlockManager#computeReplicationWork
    BlockManagerTestUtil.computeAllPendingWork(bm);

    // Initially, should have some pending replication since the close()
    // is earlier than at lease one of the reportReceivedDeletedBlocks calls
    assertTrue(pendingReplicationCount(bm) > 0);

    // release pending IBR.
    delayer.waitForCall();
    delayer.proceed();
    delayer.waitForResult();

    // make sure DataNodes do replication work if exists
    for (DataNode d : cluster.getDataNodes()) {
      DataNodeTestUtils.triggerHeartbeat(d);
    }

    // Wait until there is nothing pending
    try {
      GenericTestUtils.waitFor(new Supplier<Boolean>() {
        @Override
        public Boolean get() {
          return pendingReplicationCount(bm) == 0;
        }
      }, 100, 3000);
    } catch (TimeoutException e) {
      fail("timed out while waiting for no pending replication.");
    }

    // Check that none of the datanodes have serviced a replication request.
    // i.e. that the NameNode didn't schedule any spurious replication.
    assertNoReplicationWasPerformed(cluster);
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
  }
}
 

import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; //导入方法依赖的package包/类
/**
 * Another regression test for HDFS-2742. This tests the following sequence:
 * - DN does a block report while file is open. This BR contains
 *   the block in RBW state.
 * - The block report is delayed in reaching the standby.
 * - The file is closed.
 * - The standby processes the OP_ADD and OP_CLOSE operations before
 *   the RBW block report arrives.
 * - The standby should not mark the block as corrupt.
 */
@Test
public void testRBWReportArrivesAfterEdits() throws Exception {
  final CountDownLatch brFinished = new CountDownLatch(1);
  DelayAnswer delayer = new GenericTestUtils.DelayAnswer(LOG) {
    @Override
    protected Object passThrough(InvocationOnMock invocation)
        throws Throwable {
      try {
        return super.passThrough(invocation);
      } finally {
        // inform the test that our block report went through.
        brFinished.countDown();
      }
    }
  };

  FSDataOutputStream out = fs.create(TEST_FILE_PATH);
  try {
    AppendTestUtil.write(out, 0, 10);
    out.hflush();

    DataNode dn = cluster.getDataNodes().get(0);
    DatanodeProtocolClientSideTranslatorPB spy =
      DataNodeTestUtils.spyOnBposToNN(dn, nn2);
    
    Mockito.doAnswer(delayer)
      .when(spy).blockReport(
        Mockito.<DatanodeRegistration>anyObject(),
        Mockito.anyString(),
        Mockito.<StorageBlockReport[]>anyObject());
    dn.scheduleAllBlockReport(0);
    delayer.waitForCall();
    
  } finally {
    IOUtils.closeStream(out);
  }

  cluster.transitionToStandby(0);
  cluster.transitionToActive(1);
  
  delayer.proceed();
  brFinished.await();
  
  // Verify that no replicas are marked corrupt, and that the
  // file is readable from the failed-over standby.
  BlockManagerTestUtil.updateState(nn1.getNamesystem().getBlockManager());
  BlockManagerTestUtil.updateState(nn2.getNamesystem().getBlockManager());
  assertEquals(0, nn1.getNamesystem().getCorruptReplicaBlocks());
  assertEquals(0, nn2.getNamesystem().getCorruptReplicaBlocks());
  
  DFSTestUtil.readFile(fs, TEST_FILE_PATH);
}