Java MiniJournalCluster.shutdown方法代码示例

import org.apache.hadoop.hdfs.qjournal.MiniJournalCluster; //导入方法依赖的package包/类
/**
 * Run through the creation of a log without any faults injected,
 * and count how many RPCs are made to each node. This sets the
 * bounds for the other test cases, so they can exhaustively explore
 * the space of potential failures.
 */
private static long determineMaxIpcNumber() throws Exception {
  Configuration conf = new Configuration();
  MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf).build();
  QuorumJournalManager qjm = null;
  long ret;
  try {
    qjm = createInjectableQJM(cluster);
    qjm.format(FAKE_NSINFO);
    doWorkload(cluster, qjm);
    
    SortedSet<Integer> ipcCounts = Sets.newTreeSet();
    for (AsyncLogger l : qjm.getLoggerSetForTests().getLoggersForTests()) {
      InvocationCountingChannel ch = (InvocationCountingChannel)l;
      ch.waitForAllPendingCalls();
      ipcCounts.add(ch.getRpcCount());
    }

    // All of the loggers should have sent the same number of RPCs, since there
    // were no failures.
    assertEquals(1, ipcCounts.size());
    
    ret = ipcCounts.first();
    LOG.info("Max IPC count = " + ret);
  } finally {
    IOUtils.closeStream(qjm);
    cluster.shutdown();
  }
  return ret;
}
 

import org.apache.hadoop.hdfs.qjournal.MiniJournalCluster; //导入方法依赖的package包/类
/**
 * Run through the creation of a log without any faults injected,
 * and count how many RPCs are made to each node. This sets the
 * bounds for the other test cases, so they can exhaustively explore
 * the space of potential failures.
 */
private static long determineMaxIpcNumber() throws Exception {
  Configuration conf = new Configuration();
  MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf).build();
  cluster.waitActive();
  QuorumJournalManager qjm = null;
  long ret;
  try {
    qjm = createInjectableQJM(cluster);
    qjm.format(FAKE_NSINFO);
    doWorkload(cluster, qjm);
    
    SortedSet<Integer> ipcCounts = Sets.newTreeSet();
    for (AsyncLogger l : qjm.getLoggerSetForTests().getLoggersForTests()) {
      InvocationCountingChannel ch = (InvocationCountingChannel)l;
      ch.waitForAllPendingCalls();
      ipcCounts.add(ch.getRpcCount());
    }

    // All of the loggers should have sent the same number of RPCs, since there
    // were no failures.
    assertEquals(1, ipcCounts.size());
    
    ret = ipcCounts.first();
    LOG.info("Max IPC count = " + ret);
  } finally {
    IOUtils.closeStream(qjm);
    cluster.shutdown();
  }
  return ret;
}
 

import org.apache.hadoop.hdfs.qjournal.MiniJournalCluster; //导入方法依赖的package包/类
/**
 * Run through the creation of a log without any faults injected,
 * and count how many RPCs are made to each node. This sets the
 * bounds for the other test cases, so they can exhaustively explore
 * the space of potential failures.
 */
private static long determineMaxIpcNumber() throws Exception {
  Configuration conf = new Configuration();
  MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf).build();
  QuorumJournalManager qjm = null;
  long ret;
  try {
    qjm = createInjectableQJM(cluster);
    qjm.transitionJournal(FAKE_NSINFO, Transition.FORMAT,
        StartupOption.REGULAR);
    doWorkload(cluster, qjm);
    
    SortedSet<Integer> ipcCounts = Sets.newTreeSet();
    for (AsyncLogger l : qjm.getLoggerSetForTests().getLoggersForTests()) {
      InvocationCountingChannel ch = (InvocationCountingChannel)l;
      ch.waitForAllPendingCalls();
      ipcCounts.add(ch.getRpcCount());
    }

    // All of the loggers should have sent the same number of RPCs, since there
    // were no failures.
    assertEquals(1, ipcCounts.size());
    
    ret = ipcCounts.first();
    LOG.info("Max IPC count = " + ret);
  } finally {
    IOUtils.closeStream(qjm);
    cluster.shutdown();
  }
  return ret;
}
 

import org.apache.hadoop.hdfs.qjournal.MiniJournalCluster; //导入方法依赖的package包/类
@Test
public void testRollbackWithQJM() throws Exception {
  final Configuration conf = new HdfsConfiguration();
  MiniJournalCluster mjc = null;
  MiniDFSCluster cluster = null;
  final Path foo = new Path("/foo");
  final Path bar = new Path("/bar");

  try {
    mjc = new MiniJournalCluster.Builder(conf).numJournalNodes(
        NUM_JOURNAL_NODES).build();
    conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, mjc
        .getQuorumJournalURI(JOURNAL_ID).toString());
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    cluster.waitActive();

    DistributedFileSystem dfs = cluster.getFileSystem();
    final DFSAdmin dfsadmin = new DFSAdmin(conf);
    dfs.mkdirs(foo);

    // start rolling upgrade
    dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    Assert.assertEquals(0,
        dfsadmin.run(new String[] { "-rollingUpgrade", "prepare" }));
    dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
    // create new directory
    dfs.mkdirs(bar);
    dfs.close();

    // rollback
    cluster.restartNameNode("-rollingUpgrade", "rollback");
    // make sure /foo is still there, but /bar is not
    dfs = cluster.getFileSystem();
    Assert.assertTrue(dfs.exists(foo));
    Assert.assertFalse(dfs.exists(bar));

    // check storage in JNs
    for (int i = 0; i < NUM_JOURNAL_NODES; i++) {
      File dir = mjc.getCurrentDir(0, JOURNAL_ID);
      // segments:(startSegment, mkdir, endSegment), (startSegment, upgrade
      // marker, mkdir, endSegment)
      checkJNStorage(dir, 4, 7);
    }
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
    if (mjc != null) {
      mjc.shutdown();
    }
  }
}
 

import org.apache.hadoop.hdfs.qjournal.MiniJournalCluster; //导入方法依赖的package包/类
/**
 * Sets up two of the nodes to each drop a single RPC, at all
 * possible combinations of RPCs. This may result in the
 * active writer failing to write. After this point, a new writer
 * should be able to recover and continue writing without
 * data loss.
 */
@Test
public void testRecoverAfterDoubleFailures() throws Exception {
  final long MAX_IPC_NUMBER = determineMaxIpcNumber();
  
  for (int failA = 1; failA <= MAX_IPC_NUMBER; failA++) {
    for (int failB = 1; failB <= MAX_IPC_NUMBER; failB++) {
      String injectionStr = "(" + failA + ", " + failB + ")";
      
      LOG.info("\n\n-------------------------------------------\n" +
          "Beginning test, failing at " + injectionStr + "\n" +
          "-------------------------------------------\n\n");
      
      MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf)
        .build();
      QuorumJournalManager qjm = null;
      try {
        qjm = createInjectableQJM(cluster);
        qjm.format(FAKE_NSINFO);
        List<AsyncLogger> loggers = qjm.getLoggerSetForTests().getLoggersForTests();
        failIpcNumber(loggers.get(0), failA);
        failIpcNumber(loggers.get(1), failB);
        int lastAckedTxn = doWorkload(cluster, qjm);

        if (lastAckedTxn < 6) {
          LOG.info("Failed after injecting failures at " + injectionStr + 
              ". This is expected since we injected a failure in the " +
              "majority.");
        }
        qjm.close();
        qjm = null;

        // Now should be able to recover
        qjm = createInjectableQJM(cluster);
        long lastRecoveredTxn = QJMTestUtil.recoverAndReturnLastTxn(qjm);
        assertTrue(lastRecoveredTxn >= lastAckedTxn);
        
        writeSegment(cluster, qjm, lastRecoveredTxn + 1, 3, true);
      } catch (Throwable t) {
        // Test failure! Rethrow with the test setup info so it can be
        // easily triaged.
        throw new RuntimeException("Test failed with injection: " + injectionStr,
              t); 
      } finally {
        cluster.shutdown();
        cluster = null;
        IOUtils.closeStream(qjm);
        qjm = null;
      }
    }
  }
}
 

import org.apache.hadoop.hdfs.qjournal.MiniJournalCluster; //导入方法依赖的package包/类
@Test
public void testRollbackWithQJM() throws Exception {
  final Configuration conf = new HdfsConfiguration();
  MiniJournalCluster mjc = null;
  MiniDFSCluster cluster = null;
  final Path foo = new Path("/foo");
  final Path bar = new Path("/bar");

  try {
    mjc = new MiniJournalCluster.Builder(conf).numJournalNodes(
        NUM_JOURNAL_NODES).build();
    mjc.waitActive();
    conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, mjc
        .getQuorumJournalURI(JOURNAL_ID).toString());
    cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
    cluster.waitActive();

    DistributedFileSystem dfs = cluster.getFileSystem();
    final DFSAdmin dfsadmin = new DFSAdmin(conf);
    dfs.mkdirs(foo);

    // start rolling upgrade
    dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
    Assert.assertEquals(0,
        dfsadmin.run(new String[] { "-rollingUpgrade", "prepare" }));
    dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
    // create new directory
    dfs.mkdirs(bar);
    dfs.close();

    // rollback
    cluster.restartNameNode("-rollingUpgrade", "rollback");
    // make sure /foo is still there, but /bar is not
    dfs = cluster.getFileSystem();
    Assert.assertTrue(dfs.exists(foo));
    Assert.assertFalse(dfs.exists(bar));

    // check storage in JNs
    for (int i = 0; i < NUM_JOURNAL_NODES; i++) {
      File dir = mjc.getCurrentDir(0, JOURNAL_ID);
      // segments:(startSegment, mkdir, endSegment), (startSegment, upgrade
      // marker, mkdir, endSegment)
      checkJNStorage(dir, 4, 7);
    }
  } finally {
    if (cluster != null) {
      cluster.shutdown();
    }
    if (mjc != null) {
      mjc.shutdown();
    }
  }
}
 

import org.apache.hadoop.hdfs.qjournal.MiniJournalCluster; //导入方法依赖的package包/类
/**
 * Sets up two of the nodes to each drop a single RPC, at all
 * possible combinations of RPCs. This may result in the
 * active writer failing to write. After this point, a new writer
 * should be able to recover and continue writing without
 * data loss.
 */
@Test
public void testRecoverAfterDoubleFailures() throws Exception {
  final long MAX_IPC_NUMBER = determineMaxIpcNumber();
  
  for (int failA = 1; failA <= MAX_IPC_NUMBER; failA++) {
    for (int failB = 1; failB <= MAX_IPC_NUMBER; failB++) {
      String injectionStr = "(" + failA + ", " + failB + ")";
      
      LOG.info("\n\n-------------------------------------------\n" +
          "Beginning test, failing at " + injectionStr + "\n" +
          "-------------------------------------------\n\n");
      
      MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf)
        .build();
      cluster.waitActive();
      QuorumJournalManager qjm = null;
      try {
        qjm = createInjectableQJM(cluster);
        qjm.format(FAKE_NSINFO);
        List<AsyncLogger> loggers = qjm.getLoggerSetForTests().getLoggersForTests();
        failIpcNumber(loggers.get(0), failA);
        failIpcNumber(loggers.get(1), failB);
        int lastAckedTxn = doWorkload(cluster, qjm);

        if (lastAckedTxn < 6) {
          LOG.info("Failed after injecting failures at " + injectionStr + 
              ". This is expected since we injected a failure in the " +
              "majority.");
        }
        qjm.close();
        qjm = null;

        // Now should be able to recover
        qjm = createInjectableQJM(cluster);
        long lastRecoveredTxn = QJMTestUtil.recoverAndReturnLastTxn(qjm);
        assertTrue(lastRecoveredTxn >= lastAckedTxn);
        
        writeSegment(cluster, qjm, lastRecoveredTxn + 1, 3, true);
      } catch (Throwable t) {
        // Test failure! Rethrow with the test setup info so it can be
        // easily triaged.
        throw new RuntimeException("Test failed with injection: " + injectionStr,
              t); 
      } finally {
        cluster.shutdown();
        cluster = null;
        IOUtils.closeStream(qjm);
        qjm = null;
      }
    }
  }
}
 

import org.apache.hadoop.hdfs.qjournal.MiniJournalCluster; //导入方法依赖的package包/类
/**
 * Sets up two of the nodes to each drop a single RPC, at all
 * possible combinations of RPCs. This may result in the
 * active writer failing to write. After this point, a new writer
 * should be able to recover and continue writing without
 * data loss.
 */
@Test
public void testRecoverAfterDoubleFailures() throws Exception {
  final long MAX_IPC_NUMBER = determineMaxIpcNumber();
  
  for (int failA = 1; failA <= MAX_IPC_NUMBER; failA++) {
    for (int failB = 1; failB <= MAX_IPC_NUMBER; failB++) {
      String injectionStr = "(" + failA + ", " + failB + ")";
      
      LOG.info("\n\n-------------------------------------------\n" +
          "Beginning test, failing at " + injectionStr + "\n" +
          "-------------------------------------------\n\n");
      
      MiniJournalCluster cluster = new MiniJournalCluster.Builder(conf)
        .build();
      QuorumJournalManager qjm = null;
      try {
        qjm = createInjectableQJM(cluster);
        qjm.transitionJournal(FAKE_NSINFO, Transition.FORMAT,
            StartupOption.REGULAR);
        List<AsyncLogger> loggers = qjm.getLoggerSetForTests().getLoggersForTests();
        failIpcNumber(loggers.get(0), failA);
        failIpcNumber(loggers.get(1), failB);
        int lastAckedTxn = doWorkload(cluster, qjm);

        if (lastAckedTxn < 6) {
          LOG.info("Failed after injecting failures at " + injectionStr + 
              ". This is expected since we injected a failure in the " +
              "majority.");
        }
        qjm.close();
        qjm = null;

        // Now should be able to recover
        qjm = createInjectableQJM(cluster);
        long lastRecoveredTxn = QJMTestUtil.recoverAndReturnLastTxn(qjm);
        assertTrue(lastRecoveredTxn >= lastAckedTxn);
        
        writeSegment(cluster, qjm, lastRecoveredTxn + 1, 3, true);
      } catch (Throwable t) {
        // Test failure! Rethrow with the test setup info so it can be
        // easily triaged.
        throw new RuntimeException("Test failed with injection: " + injectionStr,
              t); 
      } finally {
        cluster.shutdown();
        cluster = null;
        IOUtils.closeStream(qjm);
        qjm = null;
      }
    }
  }
}