Java FSImageTestUtil.assertNNFilesMatch方法代码示例

import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; //导入方法依赖的package包/类
/** BootstrapStandby when the existing NN is standby */
@Test
public void testBootstrapStandbyWithStandbyNN() throws Exception {
  // make the first NN in standby state
  cluster.transitionToStandby(0);
  Configuration confNN1 = cluster.getConfiguration(1);
  
  // shut down nn1
  cluster.shutdownNameNode(1);
  
  int rc = BootstrapStandby.run(new String[] { "-force" }, confNN1);
  assertEquals(0, rc);
  
  // Should have copied over the namespace from the standby
  FSImageTestUtil.assertNNHasCheckpoints(cluster, 1,
      ImmutableList.of(0));
  FSImageTestUtil.assertNNFilesMatch(cluster);
}
 

import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; //导入方法依赖的package包/类
/** BootstrapStandby when the existing NN is active */
@Test
public void testBootstrapStandbyWithActiveNN() throws Exception {
  // make the first NN in active state
  cluster.transitionToActive(0);
  Configuration confNN1 = cluster.getConfiguration(1);
  
  // shut down nn1
  cluster.shutdownNameNode(1);
  
  int rc = BootstrapStandby.run(new String[] { "-force" }, confNN1);
  assertEquals(0, rc);
  
  // Should have copied over the namespace from the standby
  FSImageTestUtil.assertNNHasCheckpoints(cluster, 1,
      ImmutableList.of(0));
  FSImageTestUtil.assertNNFilesMatch(cluster);
}
 

import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; //导入方法依赖的package包/类
private void bootstrapStandbys() throws Exception {
  // shutdown and bootstrap all the other nns, except the first (start 1, not 0)
  for (int i = 1; i < nnCount; i++) {
    Configuration otherNNConf = cluster.getConfiguration(i);

    // shut down other nn
    cluster.shutdownNameNode(i);

    int rc = BootstrapStandby.run(new String[] { "-force" }, otherNNConf);
    assertEquals(0, rc);

    // Should have copied over the namespace from the standby
    FSImageTestUtil.assertNNHasCheckpoints(cluster, i, ImmutableList.of(0));
  }
  FSImageTestUtil.assertNNFilesMatch(cluster);
}
 

import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; //导入方法依赖的package包/类
/**
 * Test for the base success case. The primary NN
 * hasn't made any checkpoints, and we copy the fsimage_0
 * file over and start up.
 */
@Test
public void testSuccessfulBaseCase() throws Exception {
  removeStandbyNameDirs();
  
  try {
    cluster.restartNameNode(1);
    fail("Did not throw");
  } catch (IOException ioe) {
    GenericTestUtils.assertExceptionContains(
        "storage directory does not exist or is not accessible",
        ioe);
  }
  
  int rc = BootstrapStandby.run(
      new String[]{"-nonInteractive"},
      cluster.getConfiguration(1));
  assertEquals(0, rc);
  
  // Should have copied over the namespace from the active
  FSImageTestUtil.assertNNHasCheckpoints(cluster, 1,
      ImmutableList.of(0));
  FSImageTestUtil.assertNNFilesMatch(cluster);

  // We should now be able to start the standby successfully.
  cluster.restartNameNode(1);
}
 

import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; //导入方法依赖的package包/类
/**
 * Test for downloading a checkpoint made at a later checkpoint
 * from the active.
 */
@Test
public void testDownloadingLaterCheckpoint() throws Exception {
  // Roll edit logs a few times to inflate txid
  nn0.getRpcServer().rollEditLog();
  nn0.getRpcServer().rollEditLog();
  // Make checkpoint
  NameNodeAdapter.enterSafeMode(nn0, false);
  NameNodeAdapter.saveNamespace(nn0);
  NameNodeAdapter.leaveSafeMode(nn0);
  long expectedCheckpointTxId = NameNodeAdapter.getNamesystem(nn0)
    .getFSImage().getMostRecentCheckpointTxId();
  assertEquals(6, expectedCheckpointTxId);

  int rc = BootstrapStandby.run(
      new String[]{"-force"},
      cluster.getConfiguration(1));
  assertEquals(0, rc);
  
  // Should have copied over the namespace from the active
  FSImageTestUtil.assertNNHasCheckpoints(cluster, 1,
      ImmutableList.of((int)expectedCheckpointTxId));
  FSImageTestUtil.assertNNFilesMatch(cluster);

  // We should now be able to start the standby successfully.
  cluster.restartNameNode(1);
}
 

import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; //导入方法依赖的package包/类
/**
 * Test for downloading a checkpoint made at a later checkpoint
 * from the active.
 */
@Test
public void testDownloadingLaterCheckpoint() throws Exception {
  // Roll edit logs a few times to inflate txid
  nn0.getRpcServer().rollEditLog();
  nn0.getRpcServer().rollEditLog();
  // Make checkpoint
  NameNodeAdapter.enterSafeMode(nn0, false);
  NameNodeAdapter.saveNamespace(nn0);
  NameNodeAdapter.leaveSafeMode(nn0);
  long expectedCheckpointTxId = NameNodeAdapter.getNamesystem(nn0)
      .getFSImage().getMostRecentCheckpointTxId();
  assertEquals(6, expectedCheckpointTxId);

  // advance the current txid
  cluster.getFileSystem(0).create(new Path("/test_txid"), (short)1).close();

  // obtain the content of seen_txid
  URI editsUri = cluster.getSharedEditsDir(0, maxNNCount - 1);
  long seen_txid_shared = FSImageTestUtil.getStorageTxId(nn0, editsUri);

  for (int i = 1; i < maxNNCount; i++) {
    assertEquals(0, forceBootstrap(i));

    // Should have copied over the namespace from the active
    LOG.info("Checking namenode: " + i);
    FSImageTestUtil.assertNNHasCheckpoints(cluster, i,
        ImmutableList.of((int) expectedCheckpointTxId));
  }
  FSImageTestUtil.assertNNFilesMatch(cluster);

  // Make sure the seen_txid was not modified by the standby
  assertEquals(seen_txid_shared,
      FSImageTestUtil.getStorageTxId(nn0, editsUri));

  // We should now be able to start the standby successfully.
  restartNameNodesFromIndex(1);
}
 

import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; //导入方法依赖的package包/类
/**
 * Test for the base success case. The primary NN
 * hasn't made any checkpoints, and we copy the fsimage_0
 * file over and start up.
 */
@Test
public void testSuccessfulBaseCase() throws Exception {
  removeStandbyNameDirs();
  
  try {
    cluster.restartNameNode(1);
    fail("Did not throw");
  } catch (IOException ioe) {
    GenericTestUtils.assertExceptionContains(
        "Cannot start an HA namenode with name dirs that need recovery",
        ioe);
  }
  
  int rc = BootstrapStandby.run(
      new String[]{"-nonInteractive"},
      cluster.getConfiguration(1));
  assertEquals(0, rc);
  
  // Should have copied over the namespace from the active
  FSImageTestUtil.assertNNHasCheckpoints(cluster, 1,
      ImmutableList.of(0));
  FSImageTestUtil.assertNNFilesMatch(cluster);

  // We should now be able to start the standby successfully.
  cluster.restartNameNode(1);
}
 

import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; //导入方法依赖的package包/类
private void testUpgrade(UpgradeState state) throws Exception {
  cluster.transitionToActive(0);
  final Configuration confNN1 = cluster.getConfiguration(1);

  final File current = cluster.getNameNode(1).getFSImage().getStorage()
      .getStorageDir(0).getCurrentDir();
  final File tmp = cluster.getNameNode(1).getFSImage().getStorage()
      .getStorageDir(0).getPreviousTmp();
  // shut down nn1
  cluster.shutdownNameNode(1);

  // make NN0 in upgrade state
  FSImage fsImage0 = cluster.getNameNode(0).getNamesystem().getFSImage();
  Whitebox.setInternalState(fsImage0, "isUpgradeFinalized", false);

  switch (state) {
    case RECOVER:
      // rename the current directory to previous.tmp in nn1
      NNStorage.rename(current, tmp);
      break;
    case FORMAT:
      // rename the current directory to a random name so it's not formatted
      final File wrongPath = new File(current.getParentFile(), "wrong");
      NNStorage.rename(current, wrongPath);
      break;
    default:
      break;
  }

  int rc = BootstrapStandby.run(new String[] { "-force" }, confNN1);
  assertEquals(0, rc);

  // Should have copied over the namespace from the standby
  FSImageTestUtil.assertNNHasCheckpoints(cluster, 1,
      ImmutableList.of(0));
  FSImageTestUtil.assertNNFilesMatch(cluster);

  // make sure the NN1 is in upgrade state, i.e., the previous directory has
  // been successfully created
  cluster.restartNameNode(1);
  assertFalse(cluster.getNameNode(1).getNamesystem().isUpgradeFinalized());
}
 

import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; //导入方法依赖的package包/类
/**
 * While boostrapping, in_progress transaction entries should be skipped.
 * Bootstrap usage for BKJM : "-force", "-nonInteractive", "-skipSharedEditsCheck"
 */
@Test
public void testBootstrapStandbyWithActiveNN() throws Exception {
  // make nn0 active
  cluster.transitionToActive(0);
 
  // do ops and generate in-progress edit log data
  Configuration confNN1 = cluster.getConfiguration(1);
  DistributedFileSystem dfs = (DistributedFileSystem) HATestUtil
      .configureFailoverFs(cluster, confNN1);
  for (int i = 1; i <= 10; i++) {
    dfs.mkdirs(new Path("/test" + i));
  }
  dfs.close();

  // shutdown nn1 and delete its edit log files
  cluster.shutdownNameNode(1);
  deleteEditLogIfExists(confNN1);
  cluster.getNameNodeRpc(0).setSafeMode(SafeModeAction.SAFEMODE_ENTER, true);
  cluster.getNameNodeRpc(0).saveNamespace();
  cluster.getNameNodeRpc(0).setSafeMode(SafeModeAction.SAFEMODE_LEAVE, true);

  // check without -skipSharedEditsCheck, Bootstrap should fail for BKJM
  // immediately after saveNamespace
  int rc = BootstrapStandby.run(new String[] { "-force", "-nonInteractive" },
    confNN1);
  Assert.assertEquals("Mismatches return code", 6, rc);

  // check with -skipSharedEditsCheck
  rc = BootstrapStandby.run(new String[] { "-force", "-nonInteractive",
      "-skipSharedEditsCheck" }, confNN1);
  Assert.assertEquals("Mismatches return code", 0, rc);

  // Checkpoint as fast as we can, in a tight loop.
  confNN1.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, 1);
  cluster.restartNameNode(1);
  cluster.transitionToStandby(1);
 
  NameNode nn0 = cluster.getNameNode(0);
  HATestUtil.waitForStandbyToCatchUp(nn0, cluster.getNameNode(1));
  long expectedCheckpointTxId = NameNodeAdapter.getNamesystem(nn0)
      .getFSImage().getMostRecentCheckpointTxId();
  HATestUtil.waitForCheckpoint(cluster, 1,
      ImmutableList.of((int) expectedCheckpointTxId));

  // Should have copied over the namespace
  FSImageTestUtil.assertNNHasCheckpoints(cluster, 1,
      ImmutableList.of((int) expectedCheckpointTxId));
  FSImageTestUtil.assertNNFilesMatch(cluster);
}
 

import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil; //导入方法依赖的package包/类
/**
 * While boostrapping, in_progress transaction entries should be skipped.
 * Bootstrap usage for BKJM : "-force", "-nonInteractive", "-skipSharedEditsCheck"
 */
@Test
public void testBootstrapStandbyWithActiveNN() throws Exception {
  // make nn0 active
  cluster.transitionToActive(0);
 
  // do ops and generate in-progress edit log data
  Configuration confNN1 = cluster.getConfiguration(1);
  DistributedFileSystem dfs = (DistributedFileSystem) HATestUtil
      .configureFailoverFs(cluster, confNN1);
  for (int i = 1; i <= 10; i++) {
    dfs.mkdirs(new Path("/test" + i));
  }
  dfs.close();

  // shutdown nn1 and delete its edit log files
  cluster.shutdownNameNode(1);
  deleteEditLogIfExists(confNN1);
  cluster.getNameNodeRpc(0).setSafeMode(SafeModeAction.SAFEMODE_ENTER, true);
  cluster.getNameNodeRpc(0).saveNamespace(0, 0);
  cluster.getNameNodeRpc(0).setSafeMode(SafeModeAction.SAFEMODE_LEAVE, true);

  // check without -skipSharedEditsCheck, Bootstrap should fail for BKJM
  // immediately after saveNamespace
  int rc = BootstrapStandby.run(new String[] { "-force", "-nonInteractive" },
    confNN1);
  Assert.assertEquals("Mismatches return code", 6, rc);

  // check with -skipSharedEditsCheck
  rc = BootstrapStandby.run(new String[] { "-force", "-nonInteractive",
      "-skipSharedEditsCheck" }, confNN1);
  Assert.assertEquals("Mismatches return code", 0, rc);

  // Checkpoint as fast as we can, in a tight loop.
  confNN1.setInt(DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY, 1);
  cluster.restartNameNode(1);
  cluster.transitionToStandby(1);
 
  NameNode nn0 = cluster.getNameNode(0);
  HATestUtil.waitForStandbyToCatchUp(nn0, cluster.getNameNode(1));
  long expectedCheckpointTxId = NameNodeAdapter.getNamesystem(nn0)
      .getFSImage().getMostRecentCheckpointTxId();
  HATestUtil.waitForCheckpoint(cluster, 1,
      ImmutableList.of((int) expectedCheckpointTxId));

  // Should have copied over the namespace
  FSImageTestUtil.assertNNHasCheckpoints(cluster, 1,
      ImmutableList.of((int) expectedCheckpointTxId));
  FSImageTestUtil.assertNNFilesMatch(cluster);
}