Java FSEditLog类代码示例

import org.apache.hadoop.hdfs.server.namenode.FSEditLog; //导入依赖的package包/类
public void setUp(Configuration confg, MiniJournalCluster jCluster,
    String name)
    throws Exception {
  LOG.info("START TEST : " + name);
  
  handler = new TestAvatarQJMFailuresHandler();
  InjectionHandler.set(handler);

  FSEditLog.setRuntimeForTesting(Runtime.getRuntime());
  conf = confg;
  if (jCluster == null) {
    cluster = new MiniAvatarCluster.Builder(conf).numDataNodes(1)
        .enableQJM(true).build();
  } else {
    cluster = new MiniAvatarCluster.Builder(conf).numDataNodes(1)
        .enableQJM(true).setJournalCluster(jCluster).build();
  }
  fs = cluster.getFileSystem();
  journalCluster = cluster.getJournalCluster();
  
}
 

import org.apache.hadoop.hdfs.server.namenode.FSEditLog; //导入依赖的package包/类
@After
public void tearDown() throws Exception {
  try {
    super.tearDown();
    if (tempEditsFile != null) {
      if (!tempEditsFile.delete()) {
        LOG.warn("Unable to delete temporary edits file: " +
            tempEditsFile.getAbsolutePath());
      }
    }
  } finally {
    // Reset sizeFlushBuffer between each unit test (in case it has been
    // altered by a unit test to trigger a boundary condition)
    int lastSizeFlushBuffer = FSEditLog.sizeFlushBuffer;
    FSEditLog.sizeFlushBuffer = origSizeFlushBuffer;

    if (lastSizeFlushBuffer != origSizeFlushBuffer) {
      LOG.info("Setting FSEditLog.sizeFlushBuffer back to " +
          origSizeFlushBuffer + " was " + lastSizeFlushBuffer +
          " after last test.");
    }
  }
}
 

import org.apache.hadoop.hdfs.server.namenode.FSEditLog; //导入依赖的package包/类
/**
 * @see JournalManager#purgeLogsOlderThan(long)
 */
public synchronized void purgeLogsOlderThan(RequestInfo reqInfo,
    long minTxIdToKeep) throws IOException {
  checkJournalStorageFormatted();
  checkRequest(reqInfo);
  
  journalStorage.purgeDataOlderThan(minTxIdToKeep);
  if (minTxIdToKeep == FSEditLog.PURGE_ALL_TXID) {
    // When trying to remove all the segments, reset
    // the committed transaction ID too.
    committedTxnId.set(0, true);
    minTxid = 0;
  } else {
    minTxid = minTxIdToKeep;
  }
  if (imageStorage.isFormatted()) {
    imageStorage.purgeDataOlderThan(minTxIdToKeep == 0 ? -1 : minTxIdToKeep);
  }
}
 

import org.apache.hadoop.hdfs.server.namenode.FSEditLog; //导入依赖的package包/类
private LimitedEditLogAnswer causeFailureOnEditLogRead() throws IOException {
  FSEditLog spyEditLog = NameNodeAdapter.spyOnEditLog(nn1);
  LimitedEditLogAnswer answer = new LimitedEditLogAnswer(); 
  doAnswer(answer).when(spyEditLog).selectInputStreams(
      anyLong(), anyLong(), (MetaRecoveryContext)anyObject(), anyBoolean());
  return answer;
}
 

import org.apache.hadoop.hdfs.server.namenode.FSEditLog; //导入依赖的package包/类
private void processNamenodesForShutdown(Collection<Thread> threads) {
  Runtime runtime = Runtime.getRuntime();
  runtime = spy(runtime);
  doNothing().when(runtime).exit(anyInt());
  FSEditLog.setRuntimeForTesting(runtime);
  for (NameNodeInfo nnInfo : nameNodes) {
    Thread st = new Thread(new ShutDownUtil(nnInfo));
    st.start();
    threads.add(st);
  }
}
 

import org.apache.hadoop.hdfs.server.namenode.FSEditLog; //导入依赖的package包/类
private void tearDown(FSEditLog localEditLog,
    boolean writeEndTxn, boolean updateLastCorrectTxn) throws IOException {
  localEditLog.endCurrentLogSegment(writeEndTxn);
  endTxId = localEditLog.getLastWrittenTxId();
  running = false;
  lastScan = true;
  if (updateLastCorrectTxn) {
    standby.setLastCorrectTxId(endTxId);
  }
  standby.clearIngestState(endTxId + 1);
}
 

import org.apache.hadoop.hdfs.server.namenode.FSEditLog; //导入依赖的package包/类
/**
 * Tests that if the journal cluster is marked as required, we cannot
 * tolerate its failure.
 */
@Test
public void testJournalClusterFailureWhenRequired() throws Exception {
  Configuration conf = new Configuration();
  journalCluster = new MiniJournalCluster.Builder(conf).numJournalNodes(
      3).build();
  String journalURI = journalCluster.getQuorumJournalURI(
      MiniAvatarCluster.JID).toString();
  conf.set("dfs.name.edits.dir.required", journalURI + "/zero," + journalURI
      + "/one");

  setUp(conf, journalCluster, "testJournalClusterFailureWhenRequired");

  // spy the runtime
  Runtime runtime = Runtime.getRuntime();
  runtime = spy(runtime);
  doNothing().when(runtime).exit(anyInt());
  FSEditLog.setRuntimeForTesting(runtime);


  // Kill standby to ensure only 1 runtime.exit();
  cluster.killStandby();
  journalCluster.shutdown();
  fs.create(new Path("/test1"));

  // verify failure.
  verify(runtime, times(1)).exit(anyInt());
}
 

import org.apache.hadoop.hdfs.server.namenode.FSEditLog; //导入依赖的package包/类
public void setUp(String name) throws Exception {
  LOG.info("START TEST : " + name);
  
  handler = new TestAvatarQJMFailuresHandler();
  InjectionHandler.set(handler);

  FSEditLog.setRuntimeForTesting(Runtime.getRuntime());
  conf = new Configuration();
  cluster = new MiniAvatarCluster.Builder(conf).numDataNodes(1)
        .enableQJM(true).build();
  fs = cluster.getFileSystem();
  journalCluster = cluster.getJournalCluster();
}
 

import org.apache.hadoop.hdfs.server.namenode.FSEditLog; //导入依赖的package包/类
private void setUp(String name) throws Exception {
  LOG.info("------------------- test: " + name + " START ----------------");
  conf = new Configuration();
  conf.setLong("fs.checkpoint.period", 3600);

  cluster = new MiniAvatarCluster(conf, 2, true, null, null);
  fs = cluster.getFileSystem();

  // spy the runtime
  runtime = Runtime.getRuntime();
  runtime = spy(runtime);
  doNothing().when(runtime).exit(anyInt());
  FSEditLog.setRuntimeForTesting(runtime);
}
 

import org.apache.hadoop.hdfs.server.namenode.FSEditLog; //导入依赖的package包/类
@Before
public void setUp() throws Exception {
  super.setUp();

  // Remember the size of the flush buffer so that we can reset it
  // in tearDown() in case any of the tests alter it
  origSizeFlushBuffer = FSEditLog.sizeFlushBuffer;

  // Create a temp file to use for comparison of BookKeeper based
  // input stream and the file based input stream
  tempEditsFile = FSEditLogTestUtil.createTempEditsFile(
      "testBookKeeperEditLogInputStream");
}
 

import org.apache.hadoop.hdfs.server.namenode.FSEditLog; //导入依赖的package包/类
@Override
public EditLogOutputStream startLogSegment(long txId) throws IOException {
  Preconditions.checkState(isActiveWriter,
      "must recover segments before starting a new one");
  QuorumCall<AsyncLogger,Void> q = loggers.startLogSegment(txId);
  loggers.waitForWriteQuorum(q, startSegmentTimeoutMs,
      "startLogSegment(" + txId + ")");
  return new QuorumOutputStream(loggers, txId, FSEditLog.sizeFlushBuffer,
      writeTxnsTimeoutMs, metrics, journalId);
}
 

import org.apache.hadoop.hdfs.server.namenode.FSEditLog; //导入依赖的package包/类
private LimitedEditLogAnswer causeFailureOnEditLogRead() throws IOException {
  FSEditLog spyEditLog = spy(nn1.getNamesystem().getEditLogTailer()
      .getEditLog());
  LimitedEditLogAnswer answer = new LimitedEditLogAnswer(); 
  doAnswer(answer).when(spyEditLog).selectInputStreams(
      anyLong(), anyLong(), (MetaRecoveryContext)anyObject(), anyBoolean());
  nn1.getNamesystem().getEditLogTailer().setEditLog(spyEditLog);
  
  return answer;
}
 

import org.apache.hadoop.hdfs.server.namenode.FSEditLog; //导入依赖的package包/类
@VisibleForTesting
FSEditLog getEditLog() {
  return editLog;
}
 

import org.apache.hadoop.hdfs.server.namenode.FSEditLog; //导入依赖的package包/类
@VisibleForTesting
public void setEditLog(FSEditLog editLog) {
  this.editLog = editLog;
}
 

import org.apache.hadoop.hdfs.server.namenode.FSEditLog; //导入依赖的package包/类
public static void setEditLogForTesting(FSNamesystem fsn, FSEditLog newLog) {
  Whitebox.setInternalState(fsn.getFSImage(), "editLog", newLog);
  Whitebox.setInternalState(fsn.getFSDirectory(), "editLog", newLog);
}