Java EditLogOutputStream.abort方法代码示例

import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream; //导入方法依赖的package包/类
/**
 * Test the case where the NN crashes after starting a new segment
 * on all nodes, but before writing the first transaction to it.
 */
@Test
public void testCrashAtBeginningOfSegment() throws Exception {
  writeSegment(cluster, qjm, 1, 3, true);
  waitForAllPendingCalls(qjm.getLoggerSetForTests());
  
  EditLogOutputStream stm = qjm.startLogSegment(4,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  try {
    waitForAllPendingCalls(qjm.getLoggerSetForTests());
  } finally {
    stm.abort();
  }
  
  
  // Make a new QJM
  qjm = closeLater(new QuorumJournalManager(
      conf, cluster.getQuorumJournalURI(JID), FAKE_NSINFO));
  qjm.recoverUnfinalizedSegments();
  checkRecovery(cluster, 1, 3);

  writeSegment(cluster, qjm, 4, 3, true);
}
 

import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream; //导入方法依赖的package包/类
/**
 * Test the case where the NN crashes after starting a new segment
 * on all nodes, but before writing the first transaction to it.
 */
@Test
public void testCrashAtBeginningOfSegment() throws Exception {
  writeSegment(cluster, qjm, 1, 3, true);
  waitForAllPendingCalls(qjm.getLoggerSetForTests());
  
  EditLogOutputStream stm = qjm.startLogSegment(4);
  try {
    waitForAllPendingCalls(qjm.getLoggerSetForTests());
  } finally {
    stm.abort();
  }
  
  
  // Make a new QJM
  qjm = new QuorumJournalManager(
      conf, cluster.getQuorumJournalURI(JID), FAKE_NSINFO, null, false);
  qjm.recoverUnfinalizedSegments();
  checkRecovery(cluster, 1, 3);

  writeSegment(cluster, qjm, 4, 3, true);
}
 

import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream; //导入方法依赖的package包/类
/**
 * Test the case where the NN crashes after starting a new segment
 * on all nodes, but before writing the first transaction to it.
 */
@Test
public void testCrashAtBeginningOfSegment() throws Exception {
  writeSegment(cluster, qjm, 1, 3, true);
  waitForAllPendingCalls(qjm.getLoggerSetForTests());
  
  EditLogOutputStream stm = qjm.startLogSegment(4);
  try {
    waitForAllPendingCalls(qjm.getLoggerSetForTests());
  } finally {
    stm.abort();
  }
  
  
  // Make a new QJM
  qjm = closeLater(new QuorumJournalManager(
      conf, cluster.getQuorumJournalURI(JID), FAKE_NSINFO));
  qjm.recoverUnfinalizedSegments();
  checkRecovery(cluster, 1, 3);

  writeSegment(cluster, qjm, 4, 3, true);
}
 

import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream; //导入方法依赖的package包/类
@Test
public void testSimpleRecovery() throws Exception {
  NamespaceInfo nsi = newNSInfo();
  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-simplerecovery"),
      nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);;
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.setReadyToFlush();
  out.flush();

  out.abort();
  out.close();


  assertNull(zkc.exists(bkjm.finalizedLedgerZNode(1, 100), false));
  assertNotNull(zkc.exists(bkjm.inprogressZNode(1), false));

  bkjm.recoverUnfinalizedSegments();

  assertNotNull(zkc.exists(bkjm.finalizedLedgerZNode(1, 100), false));
  assertNull(zkc.exists(bkjm.inprogressZNode(1), false));
}
 

import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream; //导入方法依赖的package包/类
/**
 * Test the case where one of the loggers misses a finalizeLogSegment()
 * call, and then misses the next startLogSegment() call before coming
 * back to life.
 * 
 * Previously, this caused it to keep on writing to the old log segment,
 * such that one logger had eg edits_1-10 while the others had edits_1-5 and
 * edits_6-10. This caused recovery to fail in certain cases.
 */
@Test
public void testMissFinalizeAndNextStart() throws Exception {
  
  // Logger 0: miss finalize(1-3) and start(4)
  futureThrows(new IOException("injected")).when(spies.get(0))
    .finalizeLogSegment(Mockito.eq(1L), Mockito.eq(3L));
  futureThrows(new IOException("injected")).when(spies.get(0))
    .startLogSegment(Mockito.eq(4L));
  
  // Logger 1: fail at txn id 4
  failLoggerAtTxn(spies.get(1), 4L);
  
  writeSegment(cluster, qjm, 1, 3, true);
  EditLogOutputStream stm = qjm.startLogSegment(4);
  try {
    writeTxns(stm, 4, 1);
    fail("Did not fail to write");
  } catch (QuorumException qe) {
    // Should fail, because logger 1 had an injected fault and
    // logger 0 should detect writer out of sync
    GenericTestUtils.assertExceptionContains("Writer out of sync",
        qe);
  } finally {
    stm.abort();
    qjm.close();
  }
  
  // State:
  // Logger 0: 1-3 in-progress (since it missed finalize)
  // Logger 1: 1-3 finalized
  // Logger 2: 1-3 finalized, 4 in-progress with one txn
  
  // Shut down logger 2 so it doesn't participate in recovery
  cluster.getJournalNode(2).stopAndJoin(0);
  
  qjm = createSpyingQJM();
  long recovered = QJMTestUtil.recoverAndReturnLastTxn(qjm);
  assertEquals(3L, recovered);
}
 

import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream; //导入方法依赖的package包/类
@Test
public void testSimpleRecovery() throws Exception {
  NamespaceInfo nsi = newNSInfo();
  BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
      BKJMUtil.createJournalURI("/hdfsjournal-simplerecovery"),
      nsi);
  bkjm.format(nsi);

  EditLogOutputStream out = bkjm.startLogSegment(1);
  for (long i = 1 ; i <= 100; i++) {
    FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
    op.setTransactionId(i);
    out.write(op);
  }
  out.setReadyToFlush();
  out.flush();

  out.abort();
  out.close();


  assertNull(zkc.exists(bkjm.finalizedLedgerZNode(1, 100), false));
  assertNotNull(zkc.exists(bkjm.inprogressZNode(1), false));

  bkjm.recoverUnfinalizedSegments();

  assertNotNull(zkc.exists(bkjm.finalizedLedgerZNode(1, 100), false));
  assertNull(zkc.exists(bkjm.inprogressZNode(1), false));
}
 

import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream; //导入方法依赖的package包/类
/**
 * Test the case where, at the beginning of a segment, transactions
 * have been written to one JN but not others.
 */
public void doTestOutOfSyncAtBeginningOfSegment(int nodeWithOneTxn)
    throws Exception {
  
  int nodeWithEmptySegment = (nodeWithOneTxn + 1) % 3;
  int nodeMissingSegment = (nodeWithOneTxn + 2) % 3;
  
  writeSegment(cluster, qjm, 1, 3, true);
  waitForAllPendingCalls(qjm.getLoggerSetForTests());
  cluster.getJournalNode(nodeMissingSegment).stopAndJoin(0);
  
  // Open segment on 2/3 nodes
  EditLogOutputStream stm = qjm.startLogSegment(4,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  try {
    waitForAllPendingCalls(qjm.getLoggerSetForTests());
    
    // Write transactions to only 1/3 nodes
    failLoggerAtTxn(spies.get(nodeWithEmptySegment), 4);
    try {
      writeTxns(stm, 4, 1);
      fail("Did not fail even though 2/3 failed");
    } catch (QuorumException qe) {
      GenericTestUtils.assertExceptionContains("mock failure", qe);
    }
  } finally {
    stm.abort();
  }
  
  // Bring back the down JN.
  cluster.restartJournalNode(nodeMissingSegment);
  
  // Make a new QJM. At this point, the state is as follows:
  // A: nodeWithEmptySegment: 1-3 finalized, 4_inprogress (empty)    
  // B: nodeWithOneTxn:       1-3 finalized, 4_inprogress (1 txn)
  // C: nodeMissingSegment:   1-3 finalized
  GenericTestUtils.assertGlobEquals(
      cluster.getCurrentDir(nodeWithEmptySegment, JID),
      "edits_.*",
      NNStorage.getFinalizedEditsFileName(1, 3),
      NNStorage.getInProgressEditsFileName(4));
  GenericTestUtils.assertGlobEquals(
      cluster.getCurrentDir(nodeWithOneTxn, JID),
      "edits_.*",
      NNStorage.getFinalizedEditsFileName(1, 3),
      NNStorage.getInProgressEditsFileName(4));
  GenericTestUtils.assertGlobEquals(
      cluster.getCurrentDir(nodeMissingSegment, JID),
      "edits_.*",
      NNStorage.getFinalizedEditsFileName(1, 3));
  

  // Stop one of the nodes. Since we run this test three
  // times, rotating the roles of the nodes, we'll test
  // all the permutations.
  cluster.getJournalNode(2).stopAndJoin(0);

  qjm = createSpyingQJM();
  qjm.recoverUnfinalizedSegments();
  
  if (nodeWithOneTxn == 0 ||
      nodeWithOneTxn == 1) {
    // If the node that had the transaction committed was one of the nodes
    // that responded during recovery, then we should have recovered txid
    // 4.
    checkRecovery(cluster, 4, 4);
    writeSegment(cluster, qjm, 5, 3, true);
  } else {
    // Otherwise, we should have recovered only 1-3 and should be able to
    // start a segment at 4.
    checkRecovery(cluster, 1, 3);
    writeSegment(cluster, qjm, 4, 3, true);
  }
}
 

import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream; //导入方法依赖的package包/类
/**
 * Test the case where one of the loggers misses a finalizeLogSegment()
 * call, and then misses the next startLogSegment() call before coming
 * back to life.
 * 
 * Previously, this caused it to keep on writing to the old log segment,
 * such that one logger had eg edits_1-10 while the others had edits_1-5 and
 * edits_6-10. This caused recovery to fail in certain cases.
 */
@Test
public void testMissFinalizeAndNextStart() throws Exception {
  
  // Logger 0: miss finalize(1-3) and start(4)
  futureThrows(new IOException("injected")).when(spies.get(0))
    .finalizeLogSegment(Mockito.eq(1L), Mockito.eq(3L));
  futureThrows(new IOException("injected")).when(spies.get(0))
      .startLogSegment(Mockito.eq(4L),
          Mockito.eq(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION));
  
  // Logger 1: fail at txn id 4
  failLoggerAtTxn(spies.get(1), 4L);
  
  writeSegment(cluster, qjm, 1, 3, true);
  EditLogOutputStream stm = qjm.startLogSegment(4,
      NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
  try {
    writeTxns(stm, 4, 1);
    fail("Did not fail to write");
  } catch (QuorumException qe) {
    // Should fail, because logger 1 had an injected fault and
    // logger 0 should detect writer out of sync
    GenericTestUtils.assertExceptionContains("Writer out of sync",
        qe);
  } finally {
    stm.abort();
    qjm.close();
  }
  
  // State:
  // Logger 0: 1-3 in-progress (since it missed finalize)
  // Logger 1: 1-3 finalized
  // Logger 2: 1-3 finalized, 4 in-progress with one txn
  
  // Shut down logger 2 so it doesn't participate in recovery
  cluster.getJournalNode(2).stopAndJoin(0);
  
  qjm = createSpyingQJM();
  long recovered = QJMTestUtil.recoverAndReturnLastTxn(qjm);
  assertEquals(3L, recovered);
}
 

import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream; //导入方法依赖的package包/类
/**
 * Test the case where, at the beginning of a segment, transactions
 * have been written to one JN but not others.
 */
public void doTestOutOfSyncAtBeginningOfSegment(int nodeWithOneTxn)
    throws Exception {
  
  int nodeWithEmptySegment = (nodeWithOneTxn + 1) % 3;
  int nodeMissingSegment = (nodeWithOneTxn + 2) % 3;
  
  writeSegment(cluster, qjm, 1, 3, true);
  waitForAllPendingCalls(qjm.getLoggerSetForTests());
  cluster.getJournalNode(nodeMissingSegment).stopAndJoin(0);
  
  // Open segment on 2/3 nodes
  EditLogOutputStream stm = qjm.startLogSegment(4);
  try {
    waitForAllPendingCalls(qjm.getLoggerSetForTests());
    
    // Write transactions to only 1/3 nodes
    failLoggerAtTxn(spies.get(nodeWithEmptySegment), 4);
    try {
      writeTxns(stm, 4, 1);
      fail("Did not fail even though 2/3 failed");
    } catch (QuorumException qe) {
      GenericTestUtils.assertExceptionContains("mock failure", qe);
    }
  } finally {
    stm.abort();
  }
  
  // Bring back the down JN.
  cluster.restartJournalNode(nodeMissingSegment);
  
  // Make a new QJM. At this point, the state is as follows:
  // A: nodeWithEmptySegment: 1-3 finalized, 4_inprogress (empty)    
  // B: nodeWithOneTxn:       1-3 finalized, 4_inprogress (1 txn)
  // C: nodeMissingSegment:   1-3 finalized
  GenericTestUtils.assertGlobEquals(
      cluster.getJournalCurrentDir(nodeWithEmptySegment, JID),
      "edits_.*",
      NNStorage.getFinalizedEditsFileName(1, 3),
      NNStorage.getInProgressEditsFileName(4));
  GenericTestUtils.assertGlobEquals(
      cluster.getJournalCurrentDir(nodeWithOneTxn, JID),
      "edits_.*",
      NNStorage.getFinalizedEditsFileName(1, 3),
      NNStorage.getInProgressEditsFileName(4));
  GenericTestUtils.assertGlobEquals(
      cluster.getJournalCurrentDir(nodeMissingSegment, JID),
      "edits_.*",
      NNStorage.getFinalizedEditsFileName(1, 3));
  

  // Stop one of the nodes. Since we run this test three
  // times, rotating the roles of the nodes, we'll test
  // all the permutations.
  cluster.getJournalNode(2).stopAndJoin(0);

  qjm = createSpyingQJM();
  qjm.recoverUnfinalizedSegments();
  
  if (nodeWithOneTxn == 0 ||
      nodeWithOneTxn == 1) {
    // If the node that had the transaction committed was one of the nodes
    // that responded during recovery, then we should have recovered txid
    // 4.
    checkRecovery(cluster, 4, 4);
    writeSegment(cluster, qjm, 5, 3, true);
  } else {
    // Otherwise, we should have recovered only 1-3 and should be able to
    // start a segment at 4.
    checkRecovery(cluster, 1, 3);
    writeSegment(cluster, qjm, 4, 3, true);
  }
}
 

import org.apache.hadoop.hdfs.server.namenode.EditLogOutputStream; //导入方法依赖的package包/类
/**
 * Test the case where, at the beginning of a segment, transactions
 * have been written to one JN but not others.
 */
public void doTestOutOfSyncAtBeginningOfSegment(int nodeWithOneTxn)
    throws Exception {
  
  int nodeWithEmptySegment = (nodeWithOneTxn + 1) % 3;
  int nodeMissingSegment = (nodeWithOneTxn + 2) % 3;
  
  writeSegment(cluster, qjm, 1, 3, true);
  waitForAllPendingCalls(qjm.getLoggerSetForTests());
  cluster.getJournalNode(nodeMissingSegment).stopAndJoin(0);
  
  // Open segment on 2/3 nodes
  EditLogOutputStream stm = qjm.startLogSegment(4);
  try {
    waitForAllPendingCalls(qjm.getLoggerSetForTests());
    
    // Write transactions to only 1/3 nodes
    failLoggerAtTxn(spies.get(nodeWithEmptySegment), 4);
    try {
      writeTxns(stm, 4, 1);
      fail("Did not fail even though 2/3 failed");
    } catch (QuorumException qe) {
      GenericTestUtils.assertExceptionContains("mock failure", qe);
    }
  } finally {
    stm.abort();
  }
  
  // Bring back the down JN.
  cluster.restartJournalNode(nodeMissingSegment);
  
  // Make a new QJM. At this point, the state is as follows:
  // A: nodeWithEmptySegment: 1-3 finalized, 4_inprogress (empty)    
  // B: nodeWithOneTxn:       1-3 finalized, 4_inprogress (1 txn)
  // C: nodeMissingSegment:   1-3 finalized
  GenericTestUtils.assertGlobEquals(
      cluster.getCurrentDir(nodeWithEmptySegment, JID),
      "edits_.*",
      NNStorage.getFinalizedEditsFileName(1, 3),
      NNStorage.getInProgressEditsFileName(4));
  GenericTestUtils.assertGlobEquals(
      cluster.getCurrentDir(nodeWithOneTxn, JID),
      "edits_.*",
      NNStorage.getFinalizedEditsFileName(1, 3),
      NNStorage.getInProgressEditsFileName(4));
  GenericTestUtils.assertGlobEquals(
      cluster.getCurrentDir(nodeMissingSegment, JID),
      "edits_.*",
      NNStorage.getFinalizedEditsFileName(1, 3));
  

  // Stop one of the nodes. Since we run this test three
  // times, rotating the roles of the nodes, we'll test
  // all the permutations.
  cluster.getJournalNode(2).stopAndJoin(0);

  qjm = createSpyingQJM();
  qjm.recoverUnfinalizedSegments();
  
  if (nodeWithOneTxn == 0 ||
      nodeWithOneTxn == 1) {
    // If the node that had the transaction committed was one of the nodes
    // that responded during recovery, then we should have recovered txid
    // 4.
    checkRecovery(cluster, 4, 4);
    writeSegment(cluster, qjm, 5, 3, true);
  } else {
    // Otherwise, we should have recovered only 1-3 and should be able to
    // start a segment at 4.
    checkRecovery(cluster, 1, 3);
    writeSegment(cluster, qjm, 4, 3, true);
  }
}