Java WAL.append方法代码示例

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName,
    final byte[] family, final int count, EnvironmentEdge ee, final WAL wal,
    final HTableDescriptor htd, final MultiVersionConcurrencyControl mvcc) throws IOException {
  String familyStr = Bytes.toString(family);
  long txid = -1;
  for (int j = 0; j < count; j++) {
    byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
    byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
    WALEdit edit = new WALEdit();
    edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes));
    // uses WALKey instead of HLogKey on purpose. will only work for tests where we don't care
    // about legacy coprocessors
    txid = wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName,
        ee.currentTime(), mvcc), edit, true);
  }
  if (-1 != txid) {
    wal.sync(txid);
  }
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName,
    final byte[] family, final int count, EnvironmentEdge ee, final WAL wal,
    final HTableDescriptor htd, final MultiVersionConcurrencyControl mvcc)
throws IOException {
  String familyStr = Bytes.toString(family);
  for (int j = 0; j < count; j++) {
    byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
    byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
    WALEdit edit = new WALEdit();
    edit.add(new KeyValue(rowName, family, qualifierBytes,
      ee.currentTime(), columnBytes));
    wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName,999, mvcc),
        edit, true);
  }
  wal.sync();
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
private static long writeMarker(final WAL wal, final HTableDescriptor htd, final HRegionInfo hri,
    final WALEdit edit, final MultiVersionConcurrencyControl mvcc, final boolean sync)
throws IOException {
  // TODO: Pass in current time to use?
  WALKey key =
    new HLogKey(hri.getEncodedNameAsBytes(), hri.getTable(), System.currentTimeMillis(), mvcc);
  // Add it to the log but the false specifies that we don't need to add it to the memstore
  long trx = MultiVersionConcurrencyControl.NONE;
  try {
    trx = wal.append(htd, hri, key, edit, false);
    if (sync) wal.sync(trx);
  } finally {
    // If you get hung here, is it a real WAL or a mocked WAL? If the latter, you need to
    // trip the latch that is inside in getWriteEntry up in your mock. See down in the append
    // called from onEvent in FSHLog.
    MultiVersionConcurrencyControl.WriteEntry we = key.getWriteEntry();
    if (mvcc != null && we != null) mvcc.complete(we);
  }
  return trx;
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
/**
 * Append a faked WALEdit in order to get a long sequence number and wal syncer will just ignore
 * the WALEdit append later.
 *
 * @param wal
 * @return Return the key used appending with no sync and no append.
 * @throws IOException
 */
private WALKey appendEmptyEdit(final WAL wal) throws IOException {
  // we use HLogKey here instead of WALKey directly to support legacy
  // coprocessors.
  @SuppressWarnings("deprecation") WALKey key =
      new HLogKey(getRegionInfo().getEncodedNameAsBytes(), getRegionInfo().getTable(),
          WALKey.NO_SEQUENCE_ID, 0, null, HConstants.NO_NONCE, HConstants.NO_NONCE, getMVCC());

  // Call append but with an empty WALEdit. The returned sequence id will not
  // be associated
  // with any edit and we can be sure it went in after all outstanding
  // appends.
  try {
    wal.append(getTableDesc(), getRegionInfo(), key, WALEdit.EMPTY_WALEDIT, false);
  } catch (Throwable t) {
    // If exception, our mvcc won't get cleaned up by client, so do it here.
    getMVCC().complete(key.getWriteEntry());
  }
  return key;
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
protected void addEdits(WAL log,
                        HRegionInfo hri,
                        HTableDescriptor htd,
                        int times,
                        MultiVersionConcurrencyControl mvcc)
    throws IOException {
  final byte[] row = Bytes.toBytes("row");
  for (int i = 0; i < times; i++) {
    long timestamp = System.currentTimeMillis();
    WALEdit cols = new WALEdit();
    cols.add(new KeyValue(row, row, row, timestamp, row));
    WALKey key = new WALKey(hri.getEncodedNameAsBytes(), htd.getTableName(),
        WALKey.NO_SEQUENCE_ID, timestamp, WALKey.EMPTY_UUIDS, HConstants.NO_NONCE,
        HConstants.NO_NONCE, mvcc);
    log.append(htd, hri, key, cols, true);
  }
  log.sync();
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
/**
 * Coprocessors shouldn't get notice of empty waledits.
 */
@Test
public void testEmptyWALEditAreNotSeen() throws Exception {
  final HRegionInfo hri = createBasic3FamilyHRegionInfo(Bytes.toString(TEST_TABLE));
  final HTableDescriptor htd = createBasic3FamilyHTD(Bytes.toString(TEST_TABLE));
  final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();

  WAL log = wals.getWAL(UNSPECIFIED_REGION);
  try {
    SampleRegionWALObserver cp = getCoprocessor(log, SampleRegionWALObserver.class);

    cp.setTestValues(TEST_TABLE, null, null, null, null, null, null, null);

    assertFalse(cp.isPreWALWriteCalled());
    assertFalse(cp.isPostWALWriteCalled());

    final long now = EnvironmentEdgeManager.currentTime();
    long txid = log.append(htd, hri,
        new WALKey(hri.getEncodedNameAsBytes(), hri.getTable(), now, mvcc),
        new WALEdit(), true);
    log.sync(txid);

    assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPreWALWriteCalled());
    assertFalse("Empty WALEdit should skip coprocessor evaluation.", cp.isPostWALWriteCalled());
  } finally {
    log.close();
  }
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
/**
 * Tests the case where a RegionServer enters a GC pause,
 * comes back online after the master declared it dead and started to split.
 * Want log rolling after a master split to fail. See HBASE-2312.
 */
@Test (timeout=300000)
public void testLogRollAfterSplitStart() throws IOException {
  LOG.info("Verify wal roll after split starts will fail.");
  String logName = "testLogRollAfterSplitStart";
  Path thisTestsDir = new Path(HBASEDIR, DefaultWALProvider.getWALDirectoryName(logName));
  final WALFactory wals = new WALFactory(conf, null, logName);

  try {
    // put some entries in an WAL
    TableName tableName =
        TableName.valueOf(this.getClass().getName());
    HRegionInfo regioninfo = new HRegionInfo(tableName,
        HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
    final WAL log = wals.getWAL(regioninfo.getEncodedNameAsBytes());
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);

    final int total = 20;
    for (int i = 0; i < total; i++) {
      WALEdit kvs = new WALEdit();
      kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));
      HTableDescriptor htd = new HTableDescriptor(tableName);
      htd.addFamily(new HColumnDescriptor("column"));
      log.append(htd, regioninfo, new WALKey(regioninfo.getEncodedNameAsBytes(), tableName,
          System.currentTimeMillis(), mvcc), kvs, true);
    }
    // Send the data to HDFS datanodes and close the HDFS writer
    log.sync();
    ((FSHLog) log).replaceWriter(((FSHLog)log).getOldPath(), null, null, null);

    /* code taken from MasterFileSystem.getLogDirs(), which is called from MasterFileSystem.splitLog()
     * handles RS shutdowns (as observed by the splitting process)
     */
    // rename the directory so a rogue RS doesn't create more WALs
    Path rsSplitDir = thisTestsDir.suffix(DefaultWALProvider.SPLITTING_EXT);
    if (!fs.rename(thisTestsDir, rsSplitDir)) {
      throw new IOException("Failed fs.rename for log split: " + thisTestsDir);
    }
    LOG.debug("Renamed region directory: " + rsSplitDir);

    LOG.debug("Processing the old log files.");
    WALSplitter.split(HBASEDIR, rsSplitDir, OLDLOGDIR, fs, conf, wals);

    LOG.debug("Trying to roll the WAL.");
    try {
      log.rollWriter();
      Assert.fail("rollWriter() did not throw any exception.");
    } catch (IOException ioe) {
      if (ioe.getCause() instanceof FileNotFoundException) {
        LOG.info("Got the expected exception: ", ioe.getCause());
      } else {
        Assert.fail("Unexpected exception: " + ioe);
      }
    }
  } finally {
    wals.close();
    if (fs.exists(thisTestsDir)) {
      fs.delete(thisTestsDir, true);
    }
  }
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
/**
 * Add a bunch of dummy data and roll the logs every two insert. We
 * should end up with 10 rolled files (plus the roll called in
 * the constructor). Also test adding a listener while it's running.
 */
@Test
public void testActionListener() throws Exception {
  DummyWALActionsListener observer = new DummyWALActionsListener();
  List<WALActionsListener> list = new ArrayList<WALActionsListener>();
  list.add(observer);
  final WALFactory wals = new WALFactory(conf, list, "testActionListener");
  DummyWALActionsListener laterobserver = new DummyWALActionsListener();
  HRegionInfo hri = new HRegionInfo(TableName.valueOf(SOME_BYTES),
           SOME_BYTES, SOME_BYTES, false);
  final WAL wal = wals.getWAL(hri.getEncodedNameAsBytes());

  for (int i = 0; i < 20; i++) {
    byte[] b = Bytes.toBytes(i+"");
    KeyValue kv = new KeyValue(b,b,b);
    WALEdit edit = new WALEdit();
    edit.add(kv);
    HTableDescriptor htd = new HTableDescriptor();
    htd.addFamily(new HColumnDescriptor(b));

    final long txid = wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(),
        TableName.valueOf(b), 0), edit, true);
    wal.sync(txid);
    if (i == 10) {
      wal.registerWALActionsListener(laterobserver);
    }
    if (i % 2 == 0) {
      wal.rollWriter();
    }
  }

  wal.close();

  assertEquals(11, observer.preLogRollCounter);
  assertEquals(11, observer.postLogRollCounter);
  assertEquals(5, laterobserver.preLogRollCounter);
  assertEquals(5, laterobserver.postLogRollCounter);
  assertEquals(1, observer.closedCount);
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
@Test
public void testLogRoll() throws Exception {
  long baseline = 1000;
  long time = baseline;
  MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
  KeyValue kv = new KeyValue(r1, f1, r1);
  WALEdit edit = new WALEdit();
  edit.add(kv);

  List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
  listeners.add(replication);
  final WALFactory wals = new WALFactory(utility.getConfiguration(), listeners,
      URLEncoder.encode("regionserver:60020", "UTF8"));
  final WAL wal = wals.getWAL(hri.getEncodedNameAsBytes());
  manager.init();
  HTableDescriptor htd = new HTableDescriptor();
  htd.addFamily(new HColumnDescriptor(f1));
  // Testing normal log rolling every 20
  for(long i = 1; i < 101; i++) {
    if(i > 1 && i % 20 == 0) {
      wal.rollWriter();
    }
    LOG.info(i);
    final long txid = wal.append(htd,
        hri,
        new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc),
        edit,
        true);
    wal.sync(txid);
  }

  // Simulate a rapid insert that's followed
  // by a report that's still not totally complete (missing last one)
  LOG.info(baseline + " and " + time);
  baseline += 101;
  time = baseline;
  LOG.info(baseline + " and " + time);

  for (int i = 0; i < 3; i++) {
    wal.append(htd, hri,
        new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc),
        edit,
        true);
  }
  wal.sync();

  int logNumber = 0;
  for (Map.Entry<String, SortedSet<String>> entry : manager.getWALs().get(slaveId).entrySet()) {
    logNumber += entry.getValue().size();
  }
  assertEquals(6, logNumber);

  wal.rollWriter();

  manager.logPositionAndCleanOldLogs(manager.getSources().get(0).getCurrentPath(),
      "1", 0, false, false);

  wal.append(htd, hri,
      new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc),
      edit,
      true);
  wal.sync();

  assertEquals(1, manager.getWALs().size());


  // TODO Need a case with only 2 WALs and we only want to delete the first one
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
@Test
public void testNonLegacyWALKeysDoNotExplode() throws Exception {
  TableName tableName = TableName.valueOf(TEST_TABLE);
  final HTableDescriptor htd = createBasic3FamilyHTD(Bytes
      .toString(TEST_TABLE));
  final HRegionInfo hri = new HRegionInfo(tableName, null, null);
  MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();

  fs.mkdirs(new Path(FSUtils.getTableDir(hbaseRootDir, tableName), hri.getEncodedName()));

  final Configuration newConf = HBaseConfiguration.create(this.conf);

  final WAL wal = wals.getWAL(UNSPECIFIED_REGION);
  final SampleRegionWALObserver newApi = getCoprocessor(wal, SampleRegionWALObserver.class);
  newApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null);
  final SampleRegionWALObserver oldApi = getCoprocessor(wal,
      SampleRegionWALObserver.Legacy.class);
  oldApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null);

  LOG.debug("ensuring wal entries haven't happened before we start");
  assertFalse(newApi.isPreWALWriteCalled());
  assertFalse(newApi.isPostWALWriteCalled());
  assertFalse(newApi.isPreWALWriteDeprecatedCalled());
  assertFalse(newApi.isPostWALWriteDeprecatedCalled());
  assertFalse(oldApi.isPreWALWriteCalled());
  assertFalse(oldApi.isPostWALWriteCalled());
  assertFalse(oldApi.isPreWALWriteDeprecatedCalled());
  assertFalse(oldApi.isPostWALWriteDeprecatedCalled());

  LOG.debug("writing to WAL with non-legacy keys.");
  final int countPerFamily = 5;
  for (HColumnDescriptor hcd : htd.getFamilies()) {
    addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily,
        EnvironmentEdgeManager.getDelegate(), wal, htd, mvcc);
  }

  LOG.debug("Verify that only the non-legacy CP saw edits.");
  assertTrue(newApi.isPreWALWriteCalled());
  assertTrue(newApi.isPostWALWriteCalled());
  assertFalse(newApi.isPreWALWriteDeprecatedCalled());
  assertFalse(newApi.isPostWALWriteDeprecatedCalled());
  // wish we could test that the log message happened :/
  assertFalse(oldApi.isPreWALWriteCalled());
  assertFalse(oldApi.isPostWALWriteCalled());
  assertFalse(oldApi.isPreWALWriteDeprecatedCalled());
  assertFalse(oldApi.isPostWALWriteDeprecatedCalled());

  LOG.debug("reseting cp state.");
  newApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null);
  oldApi.setTestValues(TEST_TABLE, TEST_ROW, null, null, null, null, null, null);

  LOG.debug("write a log edit that supports legacy cps.");
  final long now = EnvironmentEdgeManager.currentTime();
  final WALKey legacyKey = new HLogKey(hri.getEncodedNameAsBytes(), hri.getTable(), now);
  final WALEdit edit = new WALEdit();
  final byte[] nonce = Bytes.toBytes("1772");
  edit.add(new KeyValue(TEST_ROW, TEST_FAMILY[0], nonce, now, nonce));
  final long txid = wal.append(htd, hri, legacyKey, edit, true);
  wal.sync(txid);

  LOG.debug("Make sure legacy cps can see supported edits after having been skipped.");
  assertTrue("non-legacy WALObserver didn't see pre-write.", newApi.isPreWALWriteCalled());
  assertTrue("non-legacy WALObserver didn't see post-write.", newApi.isPostWALWriteCalled());
  assertFalse("non-legacy WALObserver shouldn't have seen legacy pre-write.",
      newApi.isPreWALWriteDeprecatedCalled());
  assertFalse("non-legacy WALObserver shouldn't have seen legacy post-write.",
      newApi.isPostWALWriteDeprecatedCalled());
  assertTrue("legacy WALObserver didn't see pre-write.", oldApi.isPreWALWriteCalled());
  assertTrue("legacy WALObserver didn't see post-write.", oldApi.isPostWALWriteCalled());
  assertTrue("legacy WALObserver didn't see legacy pre-write.",
      oldApi.isPreWALWriteDeprecatedCalled());
  assertTrue("legacy WALObserver didn't see legacy post-write.",
      oldApi.isPostWALWriteDeprecatedCalled());
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
/**
 * Test WAL replay behavior with WALObserver.
 */
@Test
public void testWALCoprocessorReplay() throws Exception {
  // WAL replay is handled at HRegion::replayRecoveredEdits(), which is
  // ultimately called by HRegion::initialize()
  TableName tableName = TableName.valueOf("testWALCoprocessorReplay");
  final HTableDescriptor htd = getBasic3FamilyHTableDescriptor(tableName);
  MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
  // final HRegionInfo hri =
  // createBasic3FamilyHRegionInfo(Bytes.toString(tableName));
  // final HRegionInfo hri1 =
  // createBasic3FamilyHRegionInfo(Bytes.toString(tableName));
  final HRegionInfo hri = new HRegionInfo(tableName, null, null);

  final Path basedir =
      FSUtils.getTableDir(this.hbaseRootDir, tableName);
  deleteDir(basedir);
  fs.mkdirs(new Path(basedir, hri.getEncodedName()));

  final Configuration newConf = HBaseConfiguration.create(this.conf);

  // WAL wal = new WAL(this.fs, this.dir, this.oldLogDir, this.conf);
  WAL wal = wals.getWAL(UNSPECIFIED_REGION);
  // Put p = creatPutWith2Families(TEST_ROW);
  WALEdit edit = new WALEdit();
  long now = EnvironmentEdgeManager.currentTime();
  // addFamilyMapToWALEdit(p.getFamilyMap(), edit);
  final int countPerFamily = 1000;
  // for (HColumnDescriptor hcd: hri.getTableDesc().getFamilies()) {
  for (HColumnDescriptor hcd : htd.getFamilies()) {
    addWALEdits(tableName, hri, TEST_ROW, hcd.getName(), countPerFamily,
        EnvironmentEdgeManager.getDelegate(), wal, htd, mvcc);
  }
  wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName, now, mvcc), edit, true);
  // sync to fs.
  wal.sync();

  User user = HBaseTestingUtility.getDifferentUser(newConf,
      ".replay.wal.secondtime");
  user.runAs(new PrivilegedExceptionAction() {
    public Object run() throws Exception {
      Path p = runWALSplit(newConf);
      LOG.info("WALSplit path == " + p);
      FileSystem newFS = FileSystem.get(newConf);
      // Make a new wal for new region open.
      final WALFactory wals2 = new WALFactory(conf, null, currentTest.getMethodName()+"2");
      WAL wal2 = wals2.getWAL(UNSPECIFIED_REGION);;
      HRegion region = HRegion.openHRegion(newConf, FileSystem.get(newConf), hbaseRootDir,
          hri, htd, wal2, TEST_UTIL.getHBaseCluster().getRegionServer(0), null);
      long seqid2 = region.getOpenSeqNum();

      SampleRegionWALObserver cp2 =
        (SampleRegionWALObserver)region.getCoprocessorHost().findCoprocessor(
            SampleRegionWALObserver.class.getName());
      // TODO: asserting here is problematic.
      assertNotNull(cp2);
      assertTrue(cp2.isPreWALRestoreCalled());
      assertTrue(cp2.isPostWALRestoreCalled());
      assertFalse(cp2.isPreWALRestoreDeprecatedCalled());
      assertFalse(cp2.isPostWALRestoreDeprecatedCalled());
      region.close();
      wals2.close();
      return null;
    }
  });
}