Java WAL.Entry方法代码示例

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
/**
 * Test the case where the secondary region replica is not in reads enabled state because it is
 * waiting for a flush or region open marker from primary region. Replaying CANNOT_FLUSH
 * flush marker entry should restore the reads enabled status in the region and allow the reads
 * to continue.
 */
@Test
public void testReplayingFlushRequestRestoresReadsEnabledState() throws IOException {
  disableReads(secondaryRegion);

  // Test case 1: Test that replaying CANNOT_FLUSH request marker assuming this came from
  // triggered flush restores readsEnabled
  primaryRegion.flushcache(true, true);
  reader = createWALReaderForPrimary();
  while (true) {
    WAL.Entry entry = reader.next();
    if (entry == null) {
      break;
    }
    FlushDescriptor flush = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
    if (flush != null) {
      secondaryRegion.replayWALFlushMarker(flush, entry.getKey().getLogSeqNum());
    }
  }

  // now reads should be enabled
  secondaryRegion.get(new Get(Bytes.toBytes(0)));
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
private static void transformFile(Path input, Path output)
    throws IOException {
  Configuration conf = HBaseConfiguration.create();

  FileSystem inFS = input.getFileSystem(conf);
  FileSystem outFS = output.getFileSystem(conf);

  WAL.Reader in = WALFactory.createReaderIgnoreCustomClass(inFS, input, conf);
  WALProvider.Writer out = null;

  try {
    if (!(in instanceof ReaderBase)) {
      System.err.println("Cannot proceed, invalid reader type: " + in.getClass().getName());
      return;
    }
    boolean compress = ((ReaderBase)in).hasCompression();
    conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, !compress);
    out = WALFactory.createWALWriter(outFS, output, conf);

    WAL.Entry e = null;
    while ((e = in.next()) != null) out.append(e);
  } finally {
    in.close();
    if (out != null) {
      out.close();
      out = null;
    }
  }
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
/**
 * Sanity check that we can move logs around while we are reading
 * from them. Should this test fail, ReplicationSource would have a hard
 * time reading logs that are being archived.
 * @throws Exception
 */
@Test
public void testLogMoving() throws Exception{
  Path logPath = new Path(logDir, "log");
  if (!FS.exists(logDir)) FS.mkdirs(logDir);
  if (!FS.exists(oldLogDir)) FS.mkdirs(oldLogDir);
  WALProvider.Writer writer = WALFactory.createWALWriter(FS, logPath,
      TEST_UTIL.getConfiguration());
  for(int i = 0; i < 3; i++) {
    byte[] b = Bytes.toBytes(Integer.toString(i));
    KeyValue kv = new KeyValue(b,b,b);
    WALEdit edit = new WALEdit();
    edit.add(kv);
    WALKey key = new WALKey(b, TableName.valueOf(b), 0, 0,
        HConstants.DEFAULT_CLUSTER_ID);
    writer.append(new WAL.Entry(key, edit));
    writer.sync();
  }
  writer.close();

  WAL.Reader reader = WALFactory.createReader(FS, logPath, TEST_UTIL.getConfiguration());
  WAL.Entry entry = reader.next();
  assertNotNull(entry);

  Path oldLogPath = new Path(oldLogDir, "log");
  FS.rename(logPath, oldLogPath);

  entry = reader.next();
  assertNotNull(entry);

  entry = reader.next();
  entry = reader.next();

  assertNull(entry);
  reader.close();
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
@Override
public void append(WAL.Entry entry) throws IOException {
  entry.setCompressionContext(compressionContext);
  try {
    this.writer.append(entry.getKey(), entry.getEdit());
  } catch (NullPointerException npe) {
    // Concurrent close...
    throw new IOException(npe);
  }
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
/**
 * Creates a WALEdit for the passed KeyValues and returns a WALProvider.Entry instance composed of
 * the WALEdit and passed WALKey.
 * @return WAL.Entry instance for the passed WALKey and KeyValues
 */
private WAL.Entry createAEntry(WALKey walKey, List<KeyValue> kvs) {
  WALEdit edit = new WALEdit();
  for (KeyValue kv : kvs )
  edit.add(kv);
  return new WAL.Entry(walKey, edit);
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
private void verifyWALCount(WALFactory wals, WAL log, int expected) throws Exception {
  Path walPath = DefaultWALProvider.getCurrentFileName(log);
  WAL.Reader reader = wals.createReader(FS, walPath);
  int count = 0;
  WAL.Entry entry = new WAL.Entry();
  while (reader.next(entry) != null) count++;
  reader.close();
  assertEquals(expected, count);
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
/**
 * Tests a case where we replay only a flush start marker, then the region is closed. This region
 * should not block indefinitely
 */
@Test (timeout = 60000)
public void testOnlyReplayingFlushStartDoesNotHoldUpRegionClose() throws IOException {
  // load some data to primary and flush
  int start = 0;
  LOG.info("-- Writing some data to primary from " +  start + " to " + (start+100));
  putData(primaryRegion, Durability.SYNC_WAL, start, 100, cq, families);
  LOG.info("-- Flushing primary, creating 3 files for 3 stores");
  primaryRegion.flush(true);

  // now replay the edits and the flush marker
  reader = createWALReaderForPrimary();

  LOG.info("-- Replaying edits and flush events in secondary");
  while (true) {
    WAL.Entry entry = reader.next();
    if (entry == null) {
      break;
    }
    FlushDescriptor flushDesc
      = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
    if (flushDesc != null) {
      if (flushDesc.getAction() == FlushAction.START_FLUSH) {
        LOG.info("-- Replaying flush start in secondary");
        PrepareFlushResult result = secondaryRegion.replayWALFlushStartMarker(flushDesc);
      } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) {
        LOG.info("-- NOT Replaying flush commit in secondary");
      }
    } else {
      replayEdit(secondaryRegion, entry);
    }
  }

  assertTrue(rss.getRegionServerAccounting().getGlobalMemstoreSize() > 0);
  // now close the region which should not cause hold because of un-committed flush
  secondaryRegion.close();

  // verify that the memstore size is back to what it was
  assertEquals(0, rss.getRegionServerAccounting().getGlobalMemstoreSize());
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
static int replayEdit(HRegion region, WAL.Entry entry) throws IOException {
  if (WALEdit.isMetaEditFamily(entry.getEdit().getCells().get(0))) {
    return 0; // handled elsewhere
  }
  Put put = new Put(entry.getEdit().getCells().get(0).getRow());
  for (Cell cell : entry.getEdit().getCells()) put.add(cell);
  put.setDurability(Durability.SKIP_WAL);
  MutationReplay mutation = new MutationReplay(MutationType.PUT, put, 0, 0);
  region.batchReplay(new MutationReplay[] {mutation},
    entry.getKey().getLogSeqNum());
  return Integer.parseInt(Bytes.toString(put.getRow()));
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
/**
 * Tests the case where a request for flush cache is sent to the region, but region cannot flush.
 * It should write the flush request marker instead.
 */
@Test
public void testWriteFlushRequestMarker() throws IOException {
  // primary region is empty at this point. Request a flush with writeFlushRequestWalMarker=false
  FlushResultImpl result = (FlushResultImpl)((HRegion)primaryRegion).flushcache(true, false);
  assertNotNull(result);
  assertEquals(result.result, FlushResultImpl.Result.CANNOT_FLUSH_MEMSTORE_EMPTY);
  assertFalse(result.wroteFlushWalMarker);

  // request flush again, but this time with writeFlushRequestWalMarker = true
  result = (FlushResultImpl)((HRegion)primaryRegion).flushcache(true, true);
  assertNotNull(result);
  assertEquals(result.result, FlushResultImpl.Result.CANNOT_FLUSH_MEMSTORE_EMPTY);
  assertTrue(result.wroteFlushWalMarker);

  List<FlushDescriptor> flushes = Lists.newArrayList();
  reader = createWALReaderForPrimary();
  while (true) {
    WAL.Entry entry = reader.next();
    if (entry == null) {
      break;
    }
    FlushDescriptor flush = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
    if (flush != null) {
      flushes.add(flush);
    }
  }

  assertEquals(1, flushes.size());
  assertNotNull(flushes.get(0));
  assertEquals(FlushDescriptor.FlushAction.CANNOT_FLUSH, flushes.get(0).getAction());
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
/**
 * Test the case where the secondary region replica is not in reads enabled state because it is
 * waiting for a flush or region open marker from primary region. Replaying region open event
 * entry from primary should restore the reads enabled status in the region and allow the reads
 * to continue.
 */
@Test
public void testReplayingRegionOpenEventRestoresReadsEnabledState() throws IOException {
  // Test case 3: Test that replaying region open event markers restores readsEnabled
  disableReads(secondaryRegion);

  primaryRegion.close();
  primaryRegion = HRegion.openHRegion(rootDir, primaryHri, htd, walPrimary, CONF, rss, null);

  reader = createWALReaderForPrimary();
  while (true) {
    WAL.Entry entry = reader.next();
    if (entry == null) {
      break;
    }

    RegionEventDescriptor regionEventDesc
      = WALEdit.getRegionEventDescriptor(entry.getEdit().getCells().get(0));

    if (regionEventDesc != null) {
      secondaryRegion.replayWALRegionEventMarker(regionEventDesc);
    }
  }

  // now reads should be enabled
  secondaryRegion.get(new Get(Bytes.toBytes(0)));
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
/**
 * Read all the entries from the current log files and retain those that need to be replicated.
 * Else, process the end of the current file.
 * @param currentWALisBeingWrittenTo is the current WAL being written to
 * @param entries resulting entries to be replicated
 * @return true if we got nothing and went to the next file, false if we got entries
 * @throws IOException
 */
protected boolean readAllEntriesToReplicateOrNextFile(boolean currentWALisBeingWrittenTo,
    List<WAL.Entry> entries) throws IOException {
  long seenEntries = 0;
  if (LOG.isTraceEnabled()) {
    LOG.trace("Seeking in " + this.currentPath + " at position "
        + this.repLogReader.getPosition());
  }
  this.repLogReader.seek();
  long positionBeforeRead = this.repLogReader.getPosition();
  WAL.Entry entry = this.repLogReader.readNextAndSetPosition();
  while (entry != null) {
    metrics.incrLogEditsRead();
    seenEntries++;

    // don't replicate if the log entries have already been consumed by the cluster
    if (replicationEndpoint.canReplicateToSameCluster()
        || !entry.getKey().getClusterIds().contains(peerClusterId)) {
      // Remove all KVs that should not be replicated
      entry = walEntryFilter.filter(entry);
      WALEdit edit = null;
      WALKey logKey = null;
      if (entry != null) {
        edit = entry.getEdit();
        logKey = entry.getKey();
      }

      if (edit != null && edit.size() != 0) {
        // Mark that the current cluster has the change
        logKey.addClusterId(clusterId);
        currentNbOperations += countDistinctRowKeys(edit);
        entries.add(entry);
        currentSize += entry.getEdit().heapSize();
      } else {
        metrics.incrLogEditsFiltered();
      }
    }
    // Stop if too many entries or too big
    // FIXME check the relationship between single wal group and overall
    if (currentSize >= replicationQueueSizeCapacity
        || entries.size() >= replicationQueueNbCapacity) {
      break;
    }
    try {
      entry = this.repLogReader.readNextAndSetPosition();
    } catch (IOException ie) {
      LOG.debug("Break on IOE: " + ie.getMessage());
      break;
    }
  }
  metrics.incrLogReadInBytes(this.repLogReader.getPosition() - positionBeforeRead);
  if (currentWALisBeingWrittenTo) {
    return false;
  }
  // If we didn't get anything and the queue has an object, it means we
  // hit the end of the file for sure
  return seenEntries == 0 && processEndOfFile();
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
/**
 * Do the shipping logic
 * @param currentWALisBeingWrittenTo was the current WAL being (seemingly)
 * written to when this method was called
 */
protected void shipEdits(boolean currentWALisBeingWrittenTo, List<WAL.Entry> entries) {
  int sleepMultiplier = 0;
  if (entries.isEmpty()) {
    LOG.warn("Was given 0 edits to ship");
    return;
  }
  while (isWorkerActive()) {
    try {
      if (throttler.isEnabled()) {
        long sleepTicks = throttler.getNextSleepInterval(currentSize);
        if (sleepTicks > 0) {
          try {
            if (LOG.isTraceEnabled()) {
              LOG.trace("To sleep " + sleepTicks + "ms for throttling control");
            }
            Thread.sleep(sleepTicks);
          } catch (InterruptedException e) {
            LOG.debug("Interrupted while sleeping for throttling control");
            Thread.currentThread().interrupt();
            // current thread might be interrupted to terminate
            // directly go back to while() for confirm this
            continue;
          }
          // reset throttler's cycle start tick when sleep for throttling occurs
          throttler.resetStartTick();
        }
      }
      // create replicateContext here, so the entries can be GC'd upon return from this call
      // stack
      ReplicationEndpoint.ReplicateContext replicateContext =
          new ReplicationEndpoint.ReplicateContext();
      replicateContext.setEntries(entries).setSize(currentSize);
      replicateContext.setWalGroupId(walGroupId);

      long startTimeNs = System.nanoTime();
      // send the edits to the endpoint. Will block until the edits are shipped and acknowledged
      boolean replicated = replicationEndpoint.replicate(replicateContext);
      long endTimeNs = System.nanoTime();

      if (!replicated) {
        continue;
      } else {
        sleepMultiplier = Math.max(sleepMultiplier - 1, 0);
      }

      if (this.lastLoggedPosition != this.repLogReader.getPosition()) {
        manager.logPositionAndCleanOldLogs(this.currentPath, peerClusterZnode,
          this.repLogReader.getPosition(), this.replicationQueueInfo.isQueueRecovered(),
          currentWALisBeingWrittenTo);
        this.lastLoggedPosition = this.repLogReader.getPosition();
      }
      if (throttler.isEnabled()) {
        throttler.addPushSize(currentSize);
      }
      totalReplicatedEdits.addAndGet(entries.size());
      totalReplicatedOperations.addAndGet(currentNbOperations);
      // FIXME check relationship between wal group and overall
      metrics.shipBatch(currentNbOperations, currentSize / 1024);
      metrics.setAgeOfLastShippedOp(entries.get(entries.size() - 1).getKey().getWriteTime(),
        walGroupId);
      if (LOG.isTraceEnabled()) {
        LOG.trace("Replicated " + totalReplicatedEdits + " entries in total, or "
            + totalReplicatedOperations + " operations in "
            + ((endTimeNs - startTimeNs) / 1000000) + " ms");
      }
      break;
    } catch (Exception ex) {
      LOG.warn(replicationEndpoint.getClass().getName() + " threw unknown exception:"
          + org.apache.hadoop.util.StringUtils.stringifyException(ex));
      if (sleepForRetries("ReplicationEndpoint threw exception", sleepMultiplier)) {
        sleepMultiplier++;
      }
    }
  }
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
@Test
public void test() throws Exception {
  // Grab the path that was generated when the log rolled as part of its creation
  Path path = pathWatcher.currentPath;

  assertEquals(0, logManager.getPosition());

  appendToLog();

  // There's one edit in the log, read it. Reading past it needs to return nulls
  assertNotNull(logManager.openReader(path));
  logManager.seek();
  WAL.Entry entry = logManager.readNextAndSetPosition();
  assertNotNull(entry);
  entry = logManager.readNextAndSetPosition();
  assertNull(entry);
  logManager.closeReader();
  long oldPos = logManager.getPosition();

  appendToLog();

  // Read the newly added entry, make sure we made progress
  assertNotNull(logManager.openReader(path));
  logManager.seek();
  entry = logManager.readNextAndSetPosition();
  assertNotEquals(oldPos, logManager.getPosition());
  assertNotNull(entry);
  logManager.closeReader();
  oldPos = logManager.getPosition();

  log.rollWriter();

  // We rolled but we still should see the end of the first log and not get data
  assertNotNull(logManager.openReader(path));
  logManager.seek();
  entry = logManager.readNextAndSetPosition();
  assertEquals(oldPos, logManager.getPosition());
  assertNull(entry);
  logManager.finishCurrentFile();

  path = pathWatcher.currentPath;

  for (int i = 0; i < nbRows; i++) { appendToLogPlus(walEditKVs); }
  log.rollWriter();
  logManager.openReader(path);
  logManager.seek();
  for (int i = 0; i < nbRows; i++) {
    WAL.Entry e = logManager.readNextAndSetPosition();
    if (e == null) {
      fail("Should have enough entries");
    }
  }
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
/**
 * @param fs
 * @param conf
 * @param edits
 * @param region
 * @return Return how many edits seen.
 * @throws IOException
 */
private int verifyAllEditsMadeItIn(final FileSystem fs, final Configuration conf,
    final Path edits, final HRegion region)
throws IOException {
  int count = 0;
  // Based on HRegion#replayRecoveredEdits
  WAL.Reader reader = null;
  try {
    reader = WALFactory.createReader(fs, edits, conf);
    WAL.Entry entry;
    while ((entry = reader.next()) != null) {
      WALKey key = entry.getKey();
      WALEdit val = entry.getEdit();
      count++;
      // Check this edit is for this region.
      if (!Bytes.equals(key.getEncodedRegionName(),
          region.getRegionInfo().getEncodedNameAsBytes())) {
        continue;
      }
      Cell previous = null;
      for (Cell cell: val.getCells()) {
        if (CellUtil.matchingFamily(cell, WALEdit.METAFAMILY)) continue;
        if (previous != null && CellComparator.compareRows(previous, cell) == 0) continue;
        previous = cell;
        Get g = new Get(CellUtil.cloneRow(cell));
        Result r = region.get(g);
        boolean found = false;
        for (CellScanner scanner = r.cellScanner(); scanner.advance();) {
          Cell current = scanner.current();
          if (CellComparator.compare(cell, current, true) == 0) {
            found = true;
            break;
          }
        }
        assertTrue("Failed to find " + cell, found);
      }
    }
  } finally {
    if (reader != null) reader.close();
  }
  return count;
}
 

import org.apache.hadoop.hbase.wal.WAL; //导入方法依赖的package包/类
@Test
public void testReplayFlushSeqIds() throws IOException {
  // load some data to primary and flush
  int start = 0;
  LOG.info("-- Writing some data to primary from " +  start + " to " + (start+100));
  putData(primaryRegion, Durability.SYNC_WAL, start, 100, cq, families);
  LOG.info("-- Flushing primary, creating 3 files for 3 stores");
  primaryRegion.flush(true);

  // now replay the flush marker
  reader =  createWALReaderForPrimary();

  long flushSeqId = -1;
  LOG.info("-- Replaying flush events in secondary");
  while (true) {
    WAL.Entry entry = reader.next();
    if (entry == null) {
      break;
    }
    FlushDescriptor flushDesc
      = WALEdit.getFlushDescriptor(entry.getEdit().getCells().get(0));
    if (flushDesc != null) {
      if (flushDesc.getAction() == FlushAction.START_FLUSH) {
        LOG.info("-- Replaying flush start in secondary");
        secondaryRegion.replayWALFlushStartMarker(flushDesc);
        flushSeqId = flushDesc.getFlushSequenceNumber();
      } else if (flushDesc.getAction() == FlushAction.COMMIT_FLUSH) {
        LOG.info("-- Replaying flush commit in secondary");
        secondaryRegion.replayWALFlushCommitMarker(flushDesc);
        assertEquals(flushSeqId, flushDesc.getFlushSequenceNumber());
      }
    }
    // else do not replay
  }

  // TODO: what to do with this?
  // assert that the newly picked up flush file is visible
  long readPoint = secondaryRegion.getMVCC().getReadPoint();
  assertEquals(flushSeqId, readPoint);

  // after replay verify that everything is still visible
  verifyData(secondaryRegion, 0, 100, cq, families);
}