Java WALEdit.add方法代码示例

import org.apache.hadoop.hbase.regionserver.wal.WALEdit; //导入方法依赖的package包/类
private void addWALEdits(final TableName tableName, final HRegionInfo hri, final byte[] rowName,
    final byte[] family, final int count, EnvironmentEdge ee, final WAL wal,
    final HTableDescriptor htd, final MultiVersionConcurrencyControl mvcc) throws IOException {
  String familyStr = Bytes.toString(family);
  long txid = -1;
  for (int j = 0; j < count; j++) {
    byte[] qualifierBytes = Bytes.toBytes(Integer.toString(j));
    byte[] columnBytes = Bytes.toBytes(familyStr + ":" + Integer.toString(j));
    WALEdit edit = new WALEdit();
    edit.add(new KeyValue(rowName, family, qualifierBytes, ee.currentTime(), columnBytes));
    // uses WALKey instead of HLogKey on purpose. will only work for tests where we don't care
    // about legacy coprocessors
    txid = wal.append(htd, hri, new WALKey(hri.getEncodedNameAsBytes(), tableName,
        ee.currentTime(), mvcc), edit, true);
  }
  if (-1 != txid) {
    wal.sync(txid);
  }
}
 

import org.apache.hadoop.hbase.regionserver.wal.WALEdit; //导入方法依赖的package包/类
@Override
public void preBatchMutate(HRegion region, WALEdit walEdit) throws IOException {
  // TODO we should return back the status of this hook run to HRegion so that those Mutations
  // with OperationStatus as SUCCESS or FAILURE should not get applied to memstore.
  RegionCoprocessorHost coprocessorHost = region.getCoprocessorHost();
  OperationStatus[] opStatus = new OperationStatus[mutations.size()];
  Arrays.fill(opStatus, OperationStatus.NOT_RUN);
  WALEdit[] walEditsFromCP = new WALEdit[mutations.size()];
  if (coprocessorHost != null) {
    miniBatch = new MiniBatchOperationInProgress<Mutation>(
        mutations.toArray(new Mutation[mutations.size()]), opStatus, walEditsFromCP, 0,
        mutations.size());
    coprocessorHost.preBatchMutate(miniBatch);
  }
  // Apply edits to a single WALEdit
  for (int i = 0; i < mutations.size(); i++) {
    if (opStatus[i] == OperationStatus.NOT_RUN) {
      // Other OperationStatusCode means that Mutation is already succeeded or failed in CP hook
      // itself. No need to apply again to region
      if (walEditsFromCP[i] != null) {
        // Add the WALEdit created by CP hook
        for (Cell walCell : walEditsFromCP[i].getCells()) {
          walEdit.add(walCell);
        }
      }
    }
  }
}
 

import org.apache.hadoop.hbase.regionserver.wal.WALEdit; //导入方法依赖的package包/类
/**
 * Sanity check that we can move logs around while we are reading
 * from them. Should this test fail, ReplicationSource would have a hard
 * time reading logs that are being archived.
 * @throws Exception
 */
@Test
public void testLogMoving() throws Exception{
  Path logPath = new Path(logDir, "log");
  if (!FS.exists(logDir)) FS.mkdirs(logDir);
  if (!FS.exists(oldLogDir)) FS.mkdirs(oldLogDir);
  WALProvider.Writer writer = WALFactory.createWALWriter(FS, logPath,
      TEST_UTIL.getConfiguration());
  for(int i = 0; i < 3; i++) {
    byte[] b = Bytes.toBytes(Integer.toString(i));
    KeyValue kv = new KeyValue(b,b,b);
    WALEdit edit = new WALEdit();
    edit.add(kv);
    WALKey key = new WALKey(b, TableName.valueOf(b), 0, 0,
        HConstants.DEFAULT_CLUSTER_ID);
    writer.append(new WAL.Entry(key, edit));
    writer.sync();
  }
  writer.close();

  WAL.Reader reader = WALFactory.createReader(FS, logPath, TEST_UTIL.getConfiguration());
  WAL.Entry entry = reader.next();
  assertNotNull(entry);

  Path oldLogPath = new Path(oldLogDir, "log");
  FS.rename(logPath, oldLogPath);

  entry = reader.next();
  assertNotNull(entry);

  entry = reader.next();
  entry = reader.next();

  assertNull(entry);
  reader.close();
}
 

import org.apache.hadoop.hbase.regionserver.wal.WALEdit; //导入方法依赖的package包/类
private Entry createEntry(byte[]... kvs) {
  WALKey key1 = new WALKey(new byte[] {}, TableName.valueOf("foo"));
  WALEdit edit1 = new WALEdit();

  for (byte[] kv : kvs) {
    edit1.add(new KeyValue(kv, kv, kv));
  }
  return new Entry(key1, edit1);
}
 

import org.apache.hadoop.hbase.regionserver.wal.WALEdit; //导入方法依赖的package包/类
private WALEdit getWALEdits(int count) {
  WALEdit edit = new WALEdit();
  for (int i = 0; i < count; i++) {
    edit.add(new KeyValue(Bytes.toBytes(System.currentTimeMillis()), family, qualifier,
      System.currentTimeMillis(), qualifier));
  }
  return edit;
}
 

import org.apache.hadoop.hbase.regionserver.wal.WALEdit; //导入方法依赖的package包/类
private void addFamilyMapToWALEdit(Map<byte[], List<Cell>> familyMap,
    WALEdit walEdit) {
  for (List<Cell> edits : familyMap.values()) {
    for (Cell cell : edits) {
      walEdit.add(cell);
    }
  }
}
 

import org.apache.hadoop.hbase.regionserver.wal.WALEdit; //导入方法依赖的package包/类
private Path writeWAL(final WALFactory wals, final String tblName) throws IOException {
  Configuration conf = TEST_UTIL.getConfiguration();
  String clsName = conf.get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());
  conf.setClass(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, SecureWALCellCodec.class,
    WALCellCodec.class);
  try {
    TableName tableName = TableName.valueOf(tblName);
    HTableDescriptor htd = new HTableDescriptor(tableName);
    htd.addFamily(new HColumnDescriptor(tableName.getName()));
    HRegionInfo regioninfo = new HRegionInfo(tableName,
      HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false);
    final int total = 10;
    final byte[] row = Bytes.toBytes("row");
    final byte[] family = Bytes.toBytes("family");
    final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);

    // Write the WAL
    WAL wal = wals.getWAL(regioninfo.getEncodedNameAsBytes());
    for (int i = 0; i < total; i++) {
      WALEdit kvs = new WALEdit();
      kvs.add(new KeyValue(row, family, Bytes.toBytes(i), value));
      wal.append(htd, regioninfo, new WALKey(regioninfo.getEncodedNameAsBytes(), tableName,
          System.currentTimeMillis(), mvcc), kvs, true);
    }
    wal.sync();
    final Path walPath = DefaultWALProvider.getCurrentFileName(wal);
    wal.shutdown();
    
    return walPath;
  } finally {
    // restore the cell codec class
    conf.set(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, clsName);
  }
}
 

import org.apache.hadoop.hbase.regionserver.wal.WALEdit; //导入方法依赖的package包/类
protected void addEdits(WAL log, HRegionInfo hri, HTableDescriptor htd,
                      int times) throws IOException {
  final byte[] row = Bytes.toBytes("row");
  for (int i = 0; i < times; i++) {
    long timestamp = System.currentTimeMillis();
    WALEdit cols = new WALEdit();
    cols.add(new KeyValue(row, row, row, timestamp, row));
    log.append(htd, hri, getWalKey(hri.getEncodedNameAsBytes(), htd.getTableName(), timestamp),
      cols, true);
  }
  log.sync();
}
 

import org.apache.hadoop.hbase.regionserver.wal.WALEdit; //导入方法依赖的package包/类
private WAL.Entry createTestLogEntry(int i) {
  long seq = i;
  long now = i * 1000;

  WALEdit edit = new WALEdit();
  edit.add(KeyValueTestUtil.create("row", "fam", "qual", 1234, "val"));
  WALKey key = new WALKey(TEST_REGION, TEST_TABLE, seq, now,
      HConstants.DEFAULT_CLUSTER_ID);
  WAL.Entry entry = new WAL.Entry(key, edit);
  return entry;
}
 

import org.apache.hadoop.hbase.regionserver.wal.WALEdit; //导入方法依赖的package包/类
@Override
public void process(long now, HRegion region,
    List<Mutation> mutations, WALEdit walEdit) throws IOException {
  // Scan current counter
  List<Cell> kvs = new ArrayList<Cell>();
  Scan scan = new Scan(row, row);
  scan.addColumn(FAM, COUNTER);
  doScan(region, scan, kvs);
  counter = kvs.size() == 0 ? 0 :
    Bytes.toInt(CellUtil.cloneValue(kvs.iterator().next()));

  // Assert counter value
  assertEquals(expectedCounter, counter);

  // Increment counter and send it to both memstore and wal edit
  counter += 1;
  expectedCounter += 1;


  Put p = new Put(row);
  KeyValue kv =
      new KeyValue(row, FAM, COUNTER, now, Bytes.toBytes(counter));
  p.add(kv);
  mutations.add(p);
  walEdit.add(kv);

  // We can also inject some meta data to the walEdit
  KeyValue metaKv = new KeyValue(
      row, WALEdit.METAFAMILY,
      Bytes.toBytes("I just increment counter"),
      Bytes.toBytes(counter));
  walEdit.add(metaKv);
}
 

import org.apache.hadoop.hbase.regionserver.wal.WALEdit; //导入方法依赖的package包/类
/**
 * Copied from HRegion.
 * 
 * @param familyMap
 *          map of family->edits
 * @param walEdit
 *          the destination entry to append into
 */
private void addFamilyMapToWALEdit(Map<byte[], List<Cell>> familyMap,
    WALEdit walEdit) {
  for (List<Cell> edits : familyMap.values()) {
    for (Cell cell : edits) {
      // KeyValue v1 expectation. Cast for now until we go all Cell all the time. TODO.
      walEdit.add(cell);
    }
  }
}
 

import org.apache.hadoop.hbase.regionserver.wal.WALEdit; //导入方法依赖的package包/类
@Override
public boolean replicate(ReplicateContext replicateContext) {
  if (!delegator.canReplicateToSameCluster()) {
    // Only when the replication is inter cluster replication we need to covert the visibility tags to
    // string based tags.  But for intra cluster replication like region replicas it is not needed.
    List<Entry> entries = replicateContext.getEntries();
    List<Tag> visTags = new ArrayList<Tag>();
    List<Tag> nonVisTags = new ArrayList<Tag>();
    List<Entry> newEntries = new ArrayList<Entry>(entries.size());
    for (Entry entry : entries) {
      WALEdit newEdit = new WALEdit();
      ArrayList<Cell> cells = entry.getEdit().getCells();
      for (Cell cell : cells) {
        if (cell.getTagsLength() > 0) {
          visTags.clear();
          nonVisTags.clear();
          Byte serializationFormat = VisibilityUtils.extractAndPartitionTags(cell, visTags,
              nonVisTags);
          if (!visTags.isEmpty()) {
            try {
              byte[] modifiedVisExpression = visibilityLabelsService
                  .encodeVisibilityForReplication(visTags, serializationFormat);
              if (modifiedVisExpression != null) {
                nonVisTags.add(new Tag(TagType.STRING_VIS_TAG_TYPE, modifiedVisExpression));
              }
            } catch (Exception ioe) {
              LOG.error(
                  "Exception while reading the visibility labels from the cell. The replication "
                      + "would happen as per the existing format and not as string type for the cell "
                      + cell + ".", ioe);
              // just return the old entries as it is without applying the string type change
              newEdit.add(cell);
              continue;
            }
            // Recreate the cell with the new tags and the existing tags
            Cell newCell = new TagRewriteCell(cell, Tag.fromList(nonVisTags));
            newEdit.add(newCell);
          } else {
            newEdit.add(cell);
          }
        } else {
          newEdit.add(cell);
        }
      }
      newEntries.add(new Entry(entry.getKey(), newEdit));
    }
    replicateContext.setEntries(newEntries);
    return delegator.replicate(replicateContext);
  } else {
    return delegator.replicate(replicateContext);
  }
}
 

import org.apache.hadoop.hbase.regionserver.wal.WALEdit; //导入方法依赖的package包/类
@Test
public void testLogRoll() throws Exception {
  long baseline = 1000;
  long time = baseline;
  MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl();
  KeyValue kv = new KeyValue(r1, f1, r1);
  WALEdit edit = new WALEdit();
  edit.add(kv);

  List<WALActionsListener> listeners = new ArrayList<WALActionsListener>();
  listeners.add(replication);
  final WALFactory wals = new WALFactory(utility.getConfiguration(), listeners,
      URLEncoder.encode("regionserver:60020", "UTF8"));
  final WAL wal = wals.getWAL(hri.getEncodedNameAsBytes());
  manager.init();
  HTableDescriptor htd = new HTableDescriptor();
  htd.addFamily(new HColumnDescriptor(f1));
  // Testing normal log rolling every 20
  for(long i = 1; i < 101; i++) {
    if(i > 1 && i % 20 == 0) {
      wal.rollWriter();
    }
    LOG.info(i);
    final long txid = wal.append(htd,
        hri,
        new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc),
        edit,
        true);
    wal.sync(txid);
  }

  // Simulate a rapid insert that's followed
  // by a report that's still not totally complete (missing last one)
  LOG.info(baseline + " and " + time);
  baseline += 101;
  time = baseline;
  LOG.info(baseline + " and " + time);

  for (int i = 0; i < 3; i++) {
    wal.append(htd, hri,
        new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc),
        edit,
        true);
  }
  wal.sync();

  int logNumber = 0;
  for (Map.Entry<String, SortedSet<String>> entry : manager.getWALs().get(slaveId).entrySet()) {
    logNumber += entry.getValue().size();
  }
  assertEquals(6, logNumber);

  wal.rollWriter();

  manager.logPositionAndCleanOldLogs(manager.getSources().get(0).getCurrentPath(),
      "1", 0, false, false);

  wal.append(htd, hri,
      new WALKey(hri.getEncodedNameAsBytes(), test, System.currentTimeMillis(), mvcc),
      edit,
      true);
  wal.sync();

  assertEquals(1, manager.getWALs().size());


  // TODO Need a case with only 2 WALs and we only want to delete the first one
}
 

import org.apache.hadoop.hbase.regionserver.wal.WALEdit; //导入方法依赖的package包/类
/**
 * Just write multiple logs then split.  Before fix for HADOOP-2283, this
 * would fail.
 * @throws IOException
 */
@Test
public void testSplit() throws IOException {
  final TableName tableName = TableName.valueOf(currentTest.getMethodName());
  final byte [] rowName = tableName.getName();
  final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);
  final Path logdir = new Path(hbaseDir,
      DefaultWALProvider.getWALDirectoryName(currentTest.getMethodName()));
  Path oldLogDir = new Path(hbaseDir, HConstants.HREGION_OLDLOGDIR_NAME);
  final int howmany = 3;
  HRegionInfo[] infos = new HRegionInfo[3];
  Path tabledir = FSUtils.getTableDir(hbaseDir, tableName);
  fs.mkdirs(tabledir);
  for(int i = 0; i < howmany; i++) {
    infos[i] = new HRegionInfo(tableName,
              Bytes.toBytes("" + i), Bytes.toBytes("" + (i+1)), false);
    fs.mkdirs(new Path(tabledir, infos[i].getEncodedName()));
    LOG.info("allo " + new Path(tabledir, infos[i].getEncodedName()).toString());
  }
  HTableDescriptor htd = new HTableDescriptor(tableName);
  htd.addFamily(new HColumnDescriptor("column"));

  // Add edits for three regions.
  for (int ii = 0; ii < howmany; ii++) {
    for (int i = 0; i < howmany; i++) {
      final WAL log = wals.getWAL(infos[i].getEncodedNameAsBytes());
      for (int j = 0; j < howmany; j++) {
        WALEdit edit = new WALEdit();
        byte [] family = Bytes.toBytes("column");
        byte [] qualifier = Bytes.toBytes(Integer.toString(j));
        byte [] column = Bytes.toBytes("column:" + Integer.toString(j));
        edit.add(new KeyValue(rowName, family, qualifier,
            System.currentTimeMillis(), column));
        LOG.info("Region " + i + ": " + edit);
        WALKey walKey =  new WALKey(infos[i].getEncodedNameAsBytes(), tableName,
            System.currentTimeMillis(), mvcc);
        log.append(htd, infos[i], walKey, edit, true);
        walKey.getWriteEntry();
      }
      log.sync();
      log.rollWriter(true);
    }
  }
  wals.shutdown();
  List<Path> splits = WALSplitter.split(hbaseDir, logdir, oldLogDir, fs, conf, wals);
  verifySplits(splits, howmany);
}
 

import org.apache.hadoop.hbase.regionserver.wal.WALEdit; //导入方法依赖的package包/类
/**
 * Tests that we can write out an edit, close, and then read it back in again.
 * @throws IOException
 */
@Test
public void testEditAdd() throws IOException {
  final int COL_COUNT = 10;
  final HTableDescriptor htd =
      new HTableDescriptor(TableName.valueOf("tablename")).addFamily(new HColumnDescriptor(
          "column"));
  final byte [] row = Bytes.toBytes("row");
  WAL.Reader reader = null;
  try {
    final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);

    // Write columns named 1, 2, 3, etc. and then values of single byte
    // 1, 2, 3...
    long timestamp = System.currentTimeMillis();
    WALEdit cols = new WALEdit();
    for (int i = 0; i < COL_COUNT; i++) {
      cols.add(new KeyValue(row, Bytes.toBytes("column"),
          Bytes.toBytes(Integer.toString(i)),
        timestamp, new byte[] { (byte)(i + '0') }));
    }
    HRegionInfo info = new HRegionInfo(htd.getTableName(),
      row,Bytes.toBytes(Bytes.toString(row) + "1"), false);
    final WAL log = wals.getWAL(info.getEncodedNameAsBytes());

    final long txid = log.append(htd, info,
      new WALKey(info.getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis(),
          mvcc),
      cols, true);
    log.sync(txid);
    log.startCacheFlush(info.getEncodedNameAsBytes(), htd.getFamiliesKeys());
    log.completeCacheFlush(info.getEncodedNameAsBytes());
    log.shutdown();
    Path filename = DefaultWALProvider.getCurrentFileName(log);
    // Now open a reader on the log and assert append worked.
    reader = wals.createReader(fs, filename);
    // Above we added all columns on a single row so we only read one
    // entry in the below... thats why we have '1'.
    for (int i = 0; i < 1; i++) {
      WAL.Entry entry = reader.next(null);
      if (entry == null) break;
      WALKey key = entry.getKey();
      WALEdit val = entry.getEdit();
      assertTrue(Bytes.equals(info.getEncodedNameAsBytes(), key.getEncodedRegionName()));
      assertTrue(htd.getTableName().equals(key.getTablename()));
      Cell cell = val.getCells().get(0);
      assertTrue(Bytes.equals(row, cell.getRow()));
      assertEquals((byte)(i + '0'), cell.getValue()[0]);
      System.out.println(key + " " + val);
    }
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
}