Java HConstants.EMPTY_END_ROW屬性代碼示例

/**
 * <p>When scanning for a prefix the scan should stop immediately after the the last row that
 * has the specified prefix. This method calculates the closest next rowKey immediately following
 * the given rowKeyPrefix.</p>
 * <p><b>IMPORTANT: This converts a rowKey<u>Prefix</u> into a rowKey</b>.</p>
 * <p>If the prefix is an 'ASCII' string put into a byte[] then this is easy because you can
 * simply increment the last byte of the array.
 * But if your application uses real binary rowids you may run into the scenario that your
 * prefix is something like:</p>
 * &nbsp;&nbsp;&nbsp;<b>{ 0x12, 0x23, 0xFF, 0xFF }</b><br/>
 * Then this stopRow needs to be fed into the actual scan<br/>
 * &nbsp;&nbsp;&nbsp;<b>{ 0x12, 0x24 }</b> (Notice that it is shorter now)<br/>
 * This method calculates the correct stop row value for this usecase.
 *
 * @param rowKeyPrefix the rowKey<u>Prefix</u>.
 * @return the closest next rowKey immediately following the given rowKeyPrefix.
 */
private byte[] calculateTheClosestNextRowKeyForPrefix(byte[] rowKeyPrefix) {
  // Essentially we are treating it like an 'unsigned very very long' and doing +1 manually.
  // Search for the place where the trailing 0xFFs start
  int offset = rowKeyPrefix.length;
  while (offset > 0) {
    if (rowKeyPrefix[offset - 1] != (byte) 0xFF) {
      break;
    }
    offset--;
  }

  if (offset == 0) {
    // We got an 0xFFFF... (only FFs) stopRow value which is
    // the last possible prefix before the end of the table.
    // So set it to stop at the 'end of the table'
    return HConstants.EMPTY_END_ROW;
  }

  // Copy the right length of the original
  byte[] newStopRow = Arrays.copyOfRange(rowKeyPrefix, 0, offset);
  // And increment the last one
  newStopRow[newStopRow.length - 1]++;
  return newStopRow;
}
 

private HBaseScanSpec mergeScanSpecs(String functionName, HBaseScanSpec leftScanSpec, HBaseScanSpec rightScanSpec) {
  Filter newFilter = null;
  byte[] startRow = HConstants.EMPTY_START_ROW;
  byte[] stopRow = HConstants.EMPTY_END_ROW;

  switch (functionName) {
  case "booleanAnd":
    newFilter = HBaseUtils.andFilterAtIndex(
        HBaseUtils.deserializeFilter(leftScanSpec.filter),
        HBaseUtils.LAST_FILTER,
        HBaseUtils.deserializeFilter(rightScanSpec.filter));
    startRow = HBaseUtils.maxOfStartRows(leftScanSpec.startRow, rightScanSpec.startRow);
    stopRow = HBaseUtils.minOfStopRows(leftScanSpec.stopRow, rightScanSpec.stopRow);
    break;
  case "booleanOr":
    newFilter = HBaseUtils.orFilterAtIndex(
        HBaseUtils.deserializeFilter(leftScanSpec.filter),
        HBaseUtils.LAST_FILTER,
        HBaseUtils.deserializeFilter(rightScanSpec.filter));
    startRow = HBaseUtils.minOfStartRows(leftScanSpec.startRow, rightScanSpec.startRow);
    stopRow = HBaseUtils.maxOfStopRows(leftScanSpec.stopRow, rightScanSpec.stopRow);
  }
  return new HBaseScanSpec(groupScan.getTableName(), startRow, stopRow, newFilter);
}
 

private HBaseScanSpec createRowKeyPrefixScanSpec(FunctionCall call, CompareFunctionsProcessor processor) {
  byte[] startRow = processor.getRowKeyPrefixStartRow();
  byte[] stopRow  = processor.getRowKeyPrefixStopRow();
  Filter filter   = processor.getRowKeyPrefixFilter();

  if (startRow != HConstants.EMPTY_START_ROW ||
      stopRow != HConstants.EMPTY_END_ROW ||
      filter != null) {
    return new HBaseScanSpec(groupScan.getTableName(), startRow, stopRow, filter);
  }

  // else
  return null;
}
 

private List<byte[]> getStartKeysInRange(byte[] start, byte[] end)
throws IOException {
  if (start == null) {
    start = HConstants.EMPTY_START_ROW;
  }
  if (end == null) {
    end = HConstants.EMPTY_END_ROW;
  }
  return getKeysAndRegionsInRange(start, end, true).getFirst();
}
 

/**
 * Finds the stripe index for the stripe containing a row provided externally for get/scan.
 */
private final int findStripeForRow(byte[] row, boolean isStart) {
  if (isStart && row == HConstants.EMPTY_START_ROW) return 0;
  if (!isStart && row == HConstants.EMPTY_END_ROW) return state.stripeFiles.size() - 1;
  // If there's an exact match below, a stripe ends at "row". Stripe right boundary is
  // exclusive, so that means the row is in the next stripe; thus, we need to add one to index.
  // If there's no match, the return value of binarySearch is (-(insertion point) - 1), where
  // insertion point is the index of the next greater element, or list size if none. The
  // insertion point happens to be exactly what we need, so we need to add one to the result.
  return Math.abs(Arrays.binarySearch(state.stripeEndRows, row, Bytes.BYTES_COMPARATOR) + 1);
}
 

/**
 * Test for HBASE-9038, Replication.scopeWALEdits would NPE if it wasn't filtering out
 * the compaction WALEdit
 * @throws Exception
 */
@Test(timeout=300000)
public void testCompactionWALEdits() throws Exception {
  WALProtos.CompactionDescriptor compactionDescriptor =
      WALProtos.CompactionDescriptor.getDefaultInstance();
  HRegionInfo hri = new HRegionInfo(htable1.getName(),
    HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
  WALEdit edit = WALEdit.createCompaction(hri, compactionDescriptor);
  Replication.scopeWALEdits(htable1.getTableDescriptor(), new WALKey(), edit);
}
 

private Path writeWAL(final WALFactory wals, final String tblName) throws IOException {
  Configuration conf = TEST_UTIL.getConfiguration();
  String clsName = conf.get(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, WALCellCodec.class.getName());
  conf.setClass(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, SecureWALCellCodec.class,
    WALCellCodec.class);
  try {
    TableName tableName = TableName.valueOf(tblName);
    HTableDescriptor htd = new HTableDescriptor(tableName);
    htd.addFamily(new HColumnDescriptor(tableName.getName()));
    HRegionInfo regioninfo = new HRegionInfo(tableName,
      HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false);
    final int total = 10;
    final byte[] row = Bytes.toBytes("row");
    final byte[] family = Bytes.toBytes("family");
    final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);

    // Write the WAL
    WAL wal = wals.getWAL(regioninfo.getEncodedNameAsBytes());
    for (int i = 0; i < total; i++) {
      WALEdit kvs = new WALEdit();
      kvs.add(new KeyValue(row, family, Bytes.toBytes(i), value));
      wal.append(htd, regioninfo, new WALKey(regioninfo.getEncodedNameAsBytes(), tableName,
          System.currentTimeMillis(), mvcc), kvs, true);
    }
    wal.sync();
    final Path walPath = DefaultWALProvider.getCurrentFileName(wal);
    wal.shutdown();
    
    return walPath;
  } finally {
    // restore the cell codec class
    conf.set(WALCellCodec.WAL_CELL_CODEC_CLASS_KEY, clsName);
  }
}
 

@Test
public void testGetStartEndKeysWithRegionReplicas() throws IOException {
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testGetStartEndKeys"));
  HColumnDescriptor fam = new HColumnDescriptor(FAMILY);
  htd.addFamily(fam);
  byte[][] KEYS = HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE;
  HBaseAdmin admin = TEST_UTIL.getHBaseAdmin();
  admin.createTable(htd, KEYS);
  List<HRegionInfo> regions = admin.getTableRegions(htd.getTableName());

  HRegionLocator locator =
      (HRegionLocator) admin.getConnection().getRegionLocator(htd.getTableName());
  for (int regionReplication = 1; regionReplication < 4; regionReplication++) {
    List<RegionLocations> regionLocations = new ArrayList<RegionLocations>();

    // mock region locations coming from meta with multiple replicas
    for (HRegionInfo region : regions) {
      HRegionLocation[] arr = new HRegionLocation[regionReplication];
      for (int i = 0; i < arr.length; i++) {
        arr[i] = new HRegionLocation(RegionReplicaUtil.getRegionInfoForReplica(region, i), null);
      }
      regionLocations.add(new RegionLocations(arr));
    }

    Pair<byte[][], byte[][]> startEndKeys = locator.getStartEndKeys(regionLocations);

    assertEquals(KEYS.length + 1, startEndKeys.getFirst().length);

    for (int i = 0; i < KEYS.length + 1; i++) {
      byte[] startKey = i == 0 ? HConstants.EMPTY_START_ROW : KEYS[i - 1];
      byte[] endKey = i == KEYS.length ? HConstants.EMPTY_END_ROW : KEYS[i];
      assertArrayEquals(startKey, startEndKeys.getFirst()[i]);
      assertArrayEquals(endKey, startEndKeys.getSecond()[i]);
    }
  }
}
 

@Test
public void testCompactionFromPrimary() throws IOException {
  Path rootDir = new Path(dir + "testRegionReplicaSecondary");
  FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootDir);

  byte[][] families = new byte[][] {
      Bytes.toBytes("cf1"), Bytes.toBytes("cf2"), Bytes.toBytes("cf3")
  };
  byte[] cq = Bytes.toBytes("cq");
  HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("testRegionReplicaSecondary"));
  for (byte[] family : families) {
    htd.addFamily(new HColumnDescriptor(family));
  }

  long time = System.currentTimeMillis();
  HRegionInfo primaryHri = new HRegionInfo(htd.getTableName(),
    HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
    false, time, 0);
  HRegionInfo secondaryHri = new HRegionInfo(htd.getTableName(),
    HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
    false, time, 1);

  HRegion primaryRegion = null, secondaryRegion = null;

  try {
    primaryRegion = HRegion.createHRegion(primaryHri,
      rootDir, TEST_UTIL.getConfiguration(), htd);

    // load some data
    putData(primaryRegion, 0, 1000, cq, families);

    // flush region
    primaryRegion.flush(true);

    // open secondary region
    secondaryRegion = HRegion.openHRegion(rootDir, secondaryHri, htd, null, CONF);

    // move the file of the primary region to the archive, simulating a compaction
    Collection<StoreFile> storeFiles = primaryRegion.getStore(families[0]).getStorefiles();
    primaryRegion.getRegionFileSystem().removeStoreFiles(Bytes.toString(families[0]), storeFiles);
    Collection<StoreFileInfo> storeFileInfos = primaryRegion.getRegionFileSystem().getStoreFiles(families[0]);
    Assert.assertTrue(storeFileInfos == null || storeFileInfos.size() == 0);

    verifyData(secondaryRegion, 0, 1000, cq, families);
  } finally {
    if (primaryRegion != null) {
      HRegion.closeHRegion(primaryRegion);
    }
    if (secondaryRegion != null) {
      HRegion.closeHRegion(secondaryRegion);
    }
  }
}
 

private void verifyGetOrScanScenario(StripeStoreFileManager manager, boolean isGet,
    byte[] start, byte[] end, Collection<StoreFile> results) throws Exception {
  start = start != null ? start : HConstants.EMPTY_START_ROW;
  end = end != null ? end : HConstants.EMPTY_END_ROW;
  Collection<StoreFile> sfs = manager.getFilesForScanOrGet(isGet, start, end);
  assertEquals(results.size(), sfs.size());
  for (StoreFile result : results) {
    assertTrue(sfs.contains(result));
  }
}
 

/**
 * Tests the case where a RegionServer enters a GC pause,
 * comes back online after the master declared it dead and started to split.
 * Want log rolling after a master split to fail. See HBASE-2312.
 */
@Test (timeout=300000)
public void testLogRollAfterSplitStart() throws IOException {
  LOG.info("Verify wal roll after split starts will fail.");
  String logName = "testLogRollAfterSplitStart";
  Path thisTestsDir = new Path(HBASEDIR, DefaultWALProvider.getWALDirectoryName(logName));
  final WALFactory wals = new WALFactory(conf, null, logName);

  try {
    // put some entries in an WAL
    TableName tableName =
        TableName.valueOf(this.getClass().getName());
    HRegionInfo regioninfo = new HRegionInfo(tableName,
        HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
    final WAL log = wals.getWAL(regioninfo.getEncodedNameAsBytes());
    MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);

    final int total = 20;
    for (int i = 0; i < total; i++) {
      WALEdit kvs = new WALEdit();
      kvs.add(new KeyValue(Bytes.toBytes(i), tableName.getName(), tableName.getName()));
      HTableDescriptor htd = new HTableDescriptor(tableName);
      htd.addFamily(new HColumnDescriptor("column"));
      log.append(htd, regioninfo, new WALKey(regioninfo.getEncodedNameAsBytes(), tableName,
          System.currentTimeMillis(), mvcc), kvs, true);
    }
    // Send the data to HDFS datanodes and close the HDFS writer
    log.sync();
    ((FSHLog) log).replaceWriter(((FSHLog)log).getOldPath(), null, null, null);

    /* code taken from MasterFileSystem.getLogDirs(), which is called from MasterFileSystem.splitLog()
     * handles RS shutdowns (as observed by the splitting process)
     */
    // rename the directory so a rogue RS doesn't create more WALs
    Path rsSplitDir = thisTestsDir.suffix(DefaultWALProvider.SPLITTING_EXT);
    if (!fs.rename(thisTestsDir, rsSplitDir)) {
      throw new IOException("Failed fs.rename for log split: " + thisTestsDir);
    }
    LOG.debug("Renamed region directory: " + rsSplitDir);

    LOG.debug("Processing the old log files.");
    WALSplitter.split(HBASEDIR, rsSplitDir, OLDLOGDIR, fs, conf, wals);

    LOG.debug("Trying to roll the WAL.");
    try {
      log.rollWriter();
      Assert.fail("rollWriter() did not throw any exception.");
    } catch (IOException ioe) {
      if (ioe.getCause() instanceof FileNotFoundException) {
        LOG.info("Got the expected exception: ", ioe.getCause());
      } else {
        Assert.fail("Unexpected exception: " + ioe);
      }
    }
  } finally {
    wals.close();
    if (fs.exists(thisTestsDir)) {
      fs.delete(thisTestsDir, true);
    }
  }
}
 

@Before
public void setup() throws IOException {
  TEST_UTIL = HBaseTestingUtility.createLocalHTU();
  FILESYSTEM = TEST_UTIL.getTestFileSystem();
  CONF = TEST_UTIL.getConfiguration();
  dir = TEST_UTIL.getDataTestDir("TestHRegionReplayEvents").toString();
  method = name.getMethodName();
  tableName = Bytes.toBytes(name.getMethodName());
  rootDir = new Path(dir + method);
  TEST_UTIL.getConfiguration().set(HConstants.HBASE_DIR, rootDir.toString());
  method = name.getMethodName();

  htd = new HTableDescriptor(TableName.valueOf(method));
  for (byte[] family : families) {
    htd.addFamily(new HColumnDescriptor(family));
  }

  time = System.currentTimeMillis();

  primaryHri = new HRegionInfo(htd.getTableName(),
    HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
    false, time, 0);
  secondaryHri = new HRegionInfo(htd.getTableName(),
    HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW,
    false, time, 1);

  wals = TestHRegion.createWALFactory(CONF, rootDir);
  walPrimary = wals.getWAL(primaryHri.getEncodedNameAsBytes());
  walSecondary = wals.getWAL(secondaryHri.getEncodedNameAsBytes());

  rss = mock(RegionServerServices.class);
  when(rss.getServerName()).thenReturn(ServerName.valueOf("foo", 1, 1));
  when(rss.getConfiguration()).thenReturn(CONF);
  when(rss.getRegionServerAccounting()).thenReturn(new RegionServerAccounting());

  primaryRegion = HRegion.createHRegion(primaryHri, rootDir, CONF, htd, walPrimary);
  primaryRegion.close();

  primaryRegion = HRegion.openHRegion(rootDir, primaryHri, htd, walPrimary, CONF, rss, null);
  secondaryRegion = HRegion.openHRegion(secondaryHri, htd, null, CONF, rss, null);

  reader = null;
}
 

public HBaseRecordReader(
    Connection connection,
    HBaseSubScan.HBaseSubScanSpec subScanSpec,
    List<SchemaPath> projectedColumns,
    OperatorContext context,
    boolean sample) {
  super(context, projectedColumns);
  this.connection = connection;
  hbaseTableName = TableName.valueOf(
      Preconditions.checkNotNull(subScanSpec, "HBase reader needs a sub-scan spec").getTableName());
  hbaseScan = new Scan(
      subScanSpec.getStartRow() == null ? HConstants.EMPTY_START_ROW : subScanSpec.getStartRow(),
      subScanSpec.getStopRow() == null ? HConstants.EMPTY_END_ROW : subScanSpec.getStopRow());
  this.context = context;
  this.sample = sample;
  hbaseScan
      .setFilter(subScanSpec.getScanFilter())
      .setCaching((int) numRowsPerBatch);

  if (!isStarQuery()) {
    for (SchemaPath column : projectedColumns) {
      if (!column.getRootSegment().getPath().equalsIgnoreCase(ROW_KEY)) {
        NameSegment root = column.getRootSegment();
        byte[] family = root.getPath().getBytes();
        PathSegment child = root.getChild();
        if (child != null && child.isNamed()) {
          byte[] qualifier = child.getNameSegment().getPath().getBytes();
          hbaseScan.addColumn(family, qualifier);
        } else {
          hbaseScan.addFamily(family);
        }
      }
    }
  }

  /* if only the row key was requested, add a FirstKeyOnlyFilter to the scan
   * to fetch only one KV from each row. If a filter is already part of this
   * scan, add the FirstKeyOnlyFilter as the LAST filter of a MUST_PASS_ALL
   * FilterList.
   */
  if (rowKeyOnly) {
    hbaseScan.setFilter(
            HBaseUtils.andFilterAtIndex(hbaseScan.getFilter(), HBaseUtils.LAST_FILTER, new FirstKeyOnlyFilter()));
  }
}
 

/**
 * @throws IOException
 */
@Test
public void testAppend() throws IOException {
  final int COL_COUNT = 10;
  final HTableDescriptor htd =
      new HTableDescriptor(TableName.valueOf("tablename")).addFamily(new HColumnDescriptor(
          "column"));
  final byte [] row = Bytes.toBytes("row");
  WAL.Reader reader = null;
  final MultiVersionConcurrencyControl mvcc = new MultiVersionConcurrencyControl(1);
  try {
    // Write columns named 1, 2, 3, etc. and then values of single byte
    // 1, 2, 3...
    long timestamp = System.currentTimeMillis();
    WALEdit cols = new WALEdit();
    for (int i = 0; i < COL_COUNT; i++) {
      cols.add(new KeyValue(row, Bytes.toBytes("column"),
        Bytes.toBytes(Integer.toString(i)),
        timestamp, new byte[] { (byte)(i + '0') }));
    }
    HRegionInfo hri = new HRegionInfo(htd.getTableName(),
        HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW);
    final WAL log = wals.getWAL(hri.getEncodedNameAsBytes());
    final long txid = log.append(htd, hri,
      new WALKey(hri.getEncodedNameAsBytes(), htd.getTableName(), System.currentTimeMillis(),
          mvcc),
      cols, true);
    log.sync(txid);
    log.startCacheFlush(hri.getEncodedNameAsBytes(), htd.getFamiliesKeys());
    log.completeCacheFlush(hri.getEncodedNameAsBytes());
    log.shutdown();
    Path filename = DefaultWALProvider.getCurrentFileName(log);
    // Now open a reader on the log and assert append worked.
    reader = wals.createReader(fs, filename);
    WAL.Entry entry = reader.next();
    assertEquals(COL_COUNT, entry.getEdit().size());
    int idx = 0;
    for (Cell val : entry.getEdit().getCells()) {
      assertTrue(Bytes.equals(hri.getEncodedNameAsBytes(),
        entry.getKey().getEncodedRegionName()));
      assertTrue(htd.getTableName().equals(entry.getKey().getTablename()));
      assertTrue(Bytes.equals(row, val.getRow()));
      assertEquals((byte)(idx + '0'), val.getValue()[0]);
      System.out.println(entry.getKey() + " " + val);
      idx++;
    }
  } finally {
    if (reader != null) {
      reader.close();
    }
  }
}
 

@Test
public void testSecureWAL() throws Exception {
  TableName tableName = TableName.valueOf("TestSecureWAL");
  HTableDescriptor htd = new HTableDescriptor(tableName);
  htd.addFamily(new HColumnDescriptor(tableName.getName()));
  HRegionInfo regioninfo = new HRegionInfo(tableName,
    HConstants.EMPTY_START_ROW, HConstants.EMPTY_END_ROW, false);
  final int total = 10;
  final byte[] row = Bytes.toBytes("row");
  final byte[] family = Bytes.toBytes("family");
  final byte[] value = Bytes.toBytes("Test value");
  FileSystem fs = TEST_UTIL.getTestFileSystem();
  final WALFactory wals = new WALFactory(TEST_UTIL.getConfiguration(), null, "TestSecureWAL");

  // Write the WAL
  final WAL wal = wals.getWAL(regioninfo.getEncodedNameAsBytes());

  for (int i = 0; i < total; i++) {
    WALEdit kvs = new WALEdit();
    kvs.add(new KeyValue(row, family, Bytes.toBytes(i), value));
    wal.append(htd, regioninfo, new WALKey(regioninfo.getEncodedNameAsBytes(), tableName,
        System.currentTimeMillis()), kvs, true);
  }
  wal.sync();
  final Path walPath = DefaultWALProvider.getCurrentFileName(wal);
  wals.shutdown();

  // Insure edits are not plaintext
  long length = fs.getFileStatus(walPath).getLen();
  FSDataInputStream in = fs.open(walPath);
  byte[] fileData = new byte[(int)length];
  IOUtils.readFully(in, fileData);
  in.close();
  assertFalse("Cells appear to be plaintext", Bytes.contains(fileData, value));

  // Confirm the WAL can be read back
  WAL.Reader reader = wals.createReader(TEST_UTIL.getTestFileSystem(), walPath);
  int count = 0;
  WAL.Entry entry = new WAL.Entry();
  while (reader.next(entry) != null) {
    count++;
    List<Cell> cells = entry.getEdit().getCells();
    assertTrue("Should be one KV per WALEdit", cells.size() == 1);
    for (Cell cell: cells) {
      byte[] thisRow = cell.getRow();
      assertTrue("Incorrect row", Bytes.equals(thisRow, row));
      byte[] thisFamily = cell.getFamily();
      assertTrue("Incorrect family", Bytes.equals(thisFamily, family));
      byte[] thisValue = cell.getValue();
      assertTrue("Incorrect value", Bytes.equals(thisValue, value));
    }
  }
  assertEquals("Should have read back as many KVs as written", total, count);
  reader.close();
}