Java SchemaMetrics.diffMetrics方法代码示例

import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; //导入方法依赖的package包/类
private void verifyDataAndIndexBlockRead(Map<String, Long> previousMetricSnapshot,
    SchemaMetrics schemaMetrics, long expectDataBlockRead, long expectedIndexBlockRead){
  Map<String, Long> currentMetricsSnapshot = SchemaMetrics.getMetricsSnapshot();
  Map<String, Long> diffs =
    SchemaMetrics.diffMetrics(previousMetricSnapshot, currentMetricsSnapshot);

  long dataBlockRead = SchemaMetrics.getLong(diffs,
      schemaMetrics.getBlockMetricName(BlockCategory.DATA, false, BlockMetricType.READ_COUNT));
  long indexBlockRead = SchemaMetrics.getLong(diffs,
      schemaMetrics.getBlockMetricName(BlockCategory.INDEX, false, BlockMetricType.READ_COUNT));

  Assert.assertEquals(expectDataBlockRead, dataBlockRead);
  Assert.assertEquals(expectedIndexBlockRead, indexBlockRead);
}
 

import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; //导入方法依赖的package包/类
@Test
public void testScannerSelection() throws IOException {
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.setBoolean("hbase.store.delete.expired.storefile", false);
  HColumnDescriptor hcd =
    new HColumnDescriptor(FAMILY_BYTES)
        .setMaxVersions(Integer.MAX_VALUE)
        .setTimeToLive(TTL_SECONDS);
  HTableDescriptor htd = new HTableDescriptor(TABLE);
  htd.addFamily(hcd);
  HRegionInfo info = new HRegionInfo(Bytes.toBytes(TABLE));
  HRegion region =
      HRegion.createHRegion(info, TEST_UTIL.getClusterTestDir(),
          conf, htd);

  for (int iFile = 0; iFile < totalNumFiles; ++iFile) {
    if (iFile == NUM_EXPIRED_FILES) {
      Threads.sleepWithoutInterrupt(TTL_MS);
    }

    for (int iRow = 0; iRow < NUM_ROWS; ++iRow) {
      Put put = new Put(Bytes.toBytes("row" + iRow));
      for (int iCol = 0; iCol < NUM_COLS_PER_ROW; ++iCol) {
        put.add(FAMILY_BYTES, Bytes.toBytes("col" + iCol),
            Bytes.toBytes("value" + iFile + "_" + iRow + "_" + iCol));
      }
      region.put(put);
    }
    region.flushcache();
  }

  Scan scan = new Scan();
  scan.setMaxVersions(Integer.MAX_VALUE);
  CacheConfig cacheConf = new CacheConfig(conf);
  LruBlockCache cache = (LruBlockCache) cacheConf.getBlockCache();
  cache.clearCache();
  InternalScanner scanner = region.getScanner(scan);
  List<KeyValue> results = new ArrayList<KeyValue>();
  final int expectedKVsPerRow = numFreshFiles * NUM_COLS_PER_ROW;
  int numReturnedRows = 0;
  LOG.info("Scanning the entire table");
  while (scanner.next(results) || results.size() > 0) {
    assertEquals(expectedKVsPerRow, results.size());
    ++numReturnedRows;
    results.clear();
  }
  assertEquals(NUM_ROWS, numReturnedRows);
  Set<String> accessedFiles = cache.getCachedFileNamesForTest();
  LOG.debug("Files accessed during scan: " + accessedFiles);

  Map<String, Long> metricsBeforeCompaction =
    SchemaMetrics.getMetricsSnapshot();

  // Exercise both compaction codepaths.
  if (explicitCompaction) {
    region.getStore(FAMILY_BYTES).compactRecentForTesting(totalNumFiles);
  } else {
    region.compactStores();
  }

  SchemaMetrics.validateMetricChanges(metricsBeforeCompaction);
  Map<String, Long> compactionMetrics =
      SchemaMetrics.diffMetrics(metricsBeforeCompaction,
          SchemaMetrics.getMetricsSnapshot());
  long compactionDataBlocksRead = SchemaMetrics.getLong(
      compactionMetrics,
      SchemaMetrics.getInstance(TABLE, FAMILY).getBlockMetricName(
          BlockCategory.DATA, true, BlockMetricType.READ_COUNT));
  assertEquals("Invalid number of blocks accessed during compaction. " +
      "We only expect non-expired files to be accessed.",
      numFreshFiles, compactionDataBlocksRead);
  region.close();
}
 

import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; //导入方法依赖的package包/类
@Test
public void testNotCachingDataBlocksDuringCompaction() throws IOException {
  // TODO: need to change this test if we add a cache size threshold for
  // compactions, or if we implement some other kind of intelligent logic for
  // deciding what blocks to cache-on-write on compaction.
  final String table = "CompactionCacheOnWrite";
  final String cf = "myCF";
  final byte[] cfBytes = Bytes.toBytes(cf);
  final int maxVersions = 3;
  HRegion region = TEST_UTIL.createTestRegion(table, 
      new HColumnDescriptor(cf)
          .setCompressionType(compress)
          .setBloomFilterType(BLOOM_TYPE)
          .setMaxVersions(maxVersions)
          .setDataBlockEncoding(encoder.getEncodingInCache())
          .setEncodeOnDisk(encoder.getEncodingOnDisk() !=
              DataBlockEncoding.NONE)
  );
  int rowIdx = 0;
  long ts = EnvironmentEdgeManager.currentTimeMillis();
  for (int iFile = 0; iFile < 5; ++iFile) {
    for (int iRow = 0; iRow < 500; ++iRow) {
      String rowStr = "" + (rowIdx * rowIdx * rowIdx) + "row" + iFile + "_" + 
          iRow;
      Put p = new Put(Bytes.toBytes(rowStr));
      ++rowIdx;
      for (int iCol = 0; iCol < 10; ++iCol) {
        String qualStr = "col" + iCol;
        String valueStr = "value_" + rowStr + "_" + qualStr;
        for (int iTS = 0; iTS < 5; ++iTS) {
          p.add(cfBytes, Bytes.toBytes(qualStr), ts++,
              Bytes.toBytes(valueStr));
        }
      }
      region.put(p);
    }
    region.flushcache();
  }
  LruBlockCache blockCache =
      (LruBlockCache) new CacheConfig(conf).getBlockCache();
  blockCache.clearCache();
  assertEquals(0, blockCache.getBlockTypeCountsForTest().size());
  Map<String, Long> metricsBefore = SchemaMetrics.getMetricsSnapshot();
  region.compactStores();
  LOG.debug("compactStores() returned");
  SchemaMetrics.validateMetricChanges(metricsBefore);
  Map<String, Long> compactionMetrics = SchemaMetrics.diffMetrics(
      metricsBefore, SchemaMetrics.getMetricsSnapshot());
  LOG.debug(SchemaMetrics.formatMetrics(compactionMetrics));
  Map<BlockType, Integer> blockTypesInCache =
      blockCache.getBlockTypeCountsForTest();
  LOG.debug("Block types in cache: " + blockTypesInCache);
  assertNull(blockTypesInCache.get(BlockType.DATA));
  region.close();
  blockCache.shutdown();
}
 

import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; //导入方法依赖的package包/类
@Test
public void testScannerSelection() throws IOException {
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.setInt("hbase.hstore.compactionThreshold", 10000);
  HColumnDescriptor hcd = new HColumnDescriptor(FAMILY_BYTES).setBlockCacheEnabled(true)
      .setBloomFilterType(bloomType);
  HTableDescriptor htd = new HTableDescriptor(TABLE);
  htd.addFamily(hcd);
  HRegionInfo info = new HRegionInfo(Bytes.toBytes(TABLE));
  HRegion region = HRegion.createHRegion(info, TEST_UTIL.getClusterTestDir(), conf, htd);

  for (int iFile = 0; iFile < NUM_FILES; ++iFile) {
    for (int iRow = 0; iRow < NUM_ROWS; ++iRow) {
      Put put = new Put(Bytes.toBytes("row" + iRow));
      for (int iCol = 0; iCol < NUM_COLS_PER_ROW; ++iCol) {
        put.add(FAMILY_BYTES, Bytes.toBytes("col" + iCol),
            Bytes.toBytes("value" + iFile + "_" + iRow + "_" + iCol));
      }
      region.put(put);
    }
    region.flushcache();
  }

  Scan scan = new Scan(Bytes.toBytes("aaa"), Bytes.toBytes("aaz"));
  CacheConfig cacheConf = new CacheConfig(conf);
  LruBlockCache cache = (LruBlockCache) cacheConf.getBlockCache();
  cache.clearCache();
  Map<String, Long> metricsBefore = SchemaMetrics.getMetricsSnapshot();
  SchemaMetrics.validateMetricChanges(metricsBefore);
  InternalScanner scanner = region.getScanner(scan);
  List<KeyValue> results = new ArrayList<KeyValue>();
  while (scanner.next(results)) {
  }
  scanner.close();
  assertEquals(0, results.size());
  Set<String> accessedFiles = cache.getCachedFileNamesForTest();
  assertEquals(accessedFiles.size(), 0);
  //assertEquals(cache.getBlockCount(), 0);
  Map<String, Long> diffMetrics = SchemaMetrics.diffMetrics(metricsBefore,
    SchemaMetrics.getMetricsSnapshot());
  SchemaMetrics schemaMetrics = SchemaMetrics.getInstance(TABLE, FAMILY);
  long dataBlockRead = SchemaMetrics.getLong(diffMetrics,
    schemaMetrics.getBlockMetricName(BlockCategory.DATA, false, BlockMetricType.READ_COUNT));
  assertEquals(dataBlockRead, 0);
  region.close();
}
 

import org.apache.hadoop.hbase.regionserver.metrics.SchemaMetrics; //导入方法依赖的package包/类
@Test
public void testCacheBlocks() throws IOException {
  // Set index block size to be the same as normal block size.
  TEST_UTIL.getConfiguration().setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY,
      BLOCK_SIZE);

  SchemaMetrics.setUseTableNameInTest(false);
  HColumnDescriptor hcd =
      new HColumnDescriptor(Bytes.toBytes(CF))
          .setMaxVersions(MAX_VERSIONS)
          .setCompressionType(COMPRESSION_ALGORITHM)
          .setBloomFilterType(BLOOM_TYPE);
  hcd.setBlocksize(BLOCK_SIZE);
  hcd.setBlockCacheEnabled(cfCacheEnabled);
  HRegion region = TEST_UTIL.createTestRegion(TABLE, hcd);
  writeTestData(region);
  Map<String, Long> metricsBefore = SchemaMetrics.getMetricsSnapshot();
  for (int i = 0; i < NUM_ROWS; ++i) {
    Get get = new Get(Bytes.toBytes("row" + i));
    region.get(get, null);
  }
  SchemaMetrics.validateMetricChanges(metricsBefore);
  Map<String, Long> metricsAfter = SchemaMetrics.getMetricsSnapshot();
  Map<String, Long> metricsDelta = SchemaMetrics.diffMetrics(metricsBefore,
      metricsAfter);
  SchemaMetrics metrics = SchemaMetrics.getInstance(TABLE, CF);
  List<BlockCategory> importantBlockCategories =
      new ArrayList<BlockCategory>();
  importantBlockCategories.add(BlockCategory.BLOOM);
  if (hfileVersion == 2) {
    // We only have index blocks for HFile v2.
    importantBlockCategories.add(BlockCategory.INDEX);
  }

  for (BlockCategory category : importantBlockCategories) {
    String hitsMetricName = getMetricName(metrics, category);
    assertTrue("Metric " + hitsMetricName + " was not incremented",
        metricsDelta.containsKey(hitsMetricName));
    long hits = metricsDelta.get(hitsMetricName);
    assertTrue("Invalid value of " + hitsMetricName + ": " + hits, hits > 0);
  }

  if (!cfCacheEnabled) {
    // Caching is turned off for the CF, so make sure we are not caching data
    // blocks.
    String dataHitMetricName = getMetricName(metrics, BlockCategory.DATA);
    assertFalse("Nonzero value for metric " + dataHitMetricName,
        metricsDelta.containsKey(dataHitMetricName));
  }
}