Java MetricsWALSource类代码示例

import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; //导入依赖的package包/类
public MetricsRegionServerWrapperImpl(final HRegionServer regionServer) {
  this.regionServer = regionServer;
  initBlockCache();
  initMobFileCache();

  this.period =
      regionServer.conf.getLong(HConstants.REGIONSERVER_METRICS_PERIOD,
        HConstants.DEFAULT_REGIONSERVER_METRICS_PERIOD);

  this.executor = CompatibilitySingletonFactory.getInstance(MetricsExecutor.class).getExecutor();
  this.runnable = new RegionServerMetricsWrapperRunnable();
  this.executor.scheduleWithFixedDelay(this.runnable, this.period, this.period,
    TimeUnit.MILLISECONDS);
  this.metricsWALSource = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);

  try {
    this.dfsHedgedReadMetrics = FSUtils.getDFSHedgedReadMetrics(regionServer.getConfiguration());
  } catch (IOException e) {
    LOG.warn("Failed to get hedged metrics", e);
  }
  if (LOG.isInfoEnabled()) {
    LOG.info("Computing regionserver metrics every " + this.period + " milliseconds");
  }
}
 

import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; //导入依赖的package包/类
@Test
public void testBatchPutWithTsSlop() throws Exception {
  // add data with a timestamp that is too recent for range. Ensure assert
  CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
  final Put[] puts = new Put[10];
  MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);

  try {
    long syncs = prepareRegionForBachPut(puts, source, true);

    OperationStatus[] codes = this.region.batchMutate(puts);
    assertEquals(10, codes.length);
    for (int i = 0; i < 10; i++) {
      assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
    }
    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
  } finally {
    HBaseTestingUtility.closeRegionAndWAL(this.region);
    this.region = null;
  }
}
 

import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; //导入依赖的package包/类
/**
 * @return syncs initial syncTimeNumOps
 */
private long prepareRegionForBachPut(final Put[] puts, final MetricsWALSource source,
    boolean slop) throws IOException {
  this.region = initHRegion(tableName, method, CONF, COLUMN_FAMILY_BYTES);

  LOG.info("First a batch put with all valid puts");
  for (int i = 0; i < puts.length; i++) {
    puts[i] = slop ? new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100) :
        new Put(Bytes.toBytes("row_" + i));
    puts[i].addColumn(COLUMN_FAMILY_BYTES, qual, value);
  }

  long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
  metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);
  return syncs;
}
 

import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; //导入依赖的package包/类
@Test
public void testBatchPut_whileNoRowLocksHeld() throws IOException {
  byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
  byte[] qual = Bytes.toBytes("qual");
  byte[] val = Bytes.toBytes("val");
  this.region = initHRegion(Bytes.toBytes(getName()), getName(), CONF, cf);
  MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
  try {
    long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);

    LOG.info("First a batch put with all valid puts");
    final Put[] puts = new Put[10];
    for (int i = 0; i < 10; i++) {
      puts[i] = new Put(Bytes.toBytes("row_" + i));
      puts[i].add(cf, qual, val);
    }

    OperationStatus[] codes = this.region.batchMutate(puts);
    assertEquals(10, codes.length);
    for (int i = 0; i < 10; i++) {
      assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
    }
    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);

    LOG.info("Next a batch put with one invalid family");
    puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
    codes = this.region.batchMutate(puts);
    assertEquals(10, codes.length);
    for (int i = 0; i < 10; i++) {
      assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
          codes[i].getOperationStatusCode());
    }

    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
  } finally {
    HRegion.closeHRegion(this.region);
    this.region = null;
  }
}
 

import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; //导入依赖的package包/类
private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount)
    throws InterruptedException {
  long startWait = System.currentTimeMillis();
  long currentCount;
  while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) {
    Thread.sleep(100);
    if (System.currentTimeMillis() - startWait > 10000) {
      fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName,
        expectedCount, currentCount));
    }
  }
}
 

import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; //导入依赖的package包/类
@Test
public void testBatchPutWithTsSlop() throws Exception {
  byte[] b = Bytes.toBytes(getName());
  byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
  byte[] qual = Bytes.toBytes("qual");
  byte[] val = Bytes.toBytes("val");

  // add data with a timestamp that is too recent for range. Ensure assert
  CONF.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
  this.region = initHRegion(b, getName(), CONF, cf);

  try {
    MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
    long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);

    final Put[] puts = new Put[10];
    for (int i = 0; i < 10; i++) {
      puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
      puts[i].add(cf, qual, val);
    }

    OperationStatus[] codes = this.region.batchMutate(puts);
    assertEquals(10, codes.length);
    for (int i = 0; i < 10; i++) {
      assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
    }
    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);

  } finally {
    HRegion.closeHRegion(this.region);
    this.region = null;
  }

}
 

import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; //导入依赖的package包/类
@Test
public void testBatchPut_whileNoRowLocksHeld() throws IOException {
  final Put[] puts = new Put[10];
  MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
  try {
    long syncs = prepareRegionForBachPut(puts, source, false);

    OperationStatus[] codes = this.region.batchMutate(puts);
    assertEquals(10, codes.length);
    for (int i = 0; i < 10; i++) {
      assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
    }
    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);

    LOG.info("Next a batch put with one invalid family");
    puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value);
    codes = this.region.batchMutate(puts);
    assertEquals(10, codes.length);
    for (int i = 0; i < 10; i++) {
      assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
          codes[i].getOperationStatusCode());
    }

    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);
  } finally {
    HBaseTestingUtility.closeRegionAndWAL(this.region);
    this.region = null;
  }
}
 

import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; //导入依赖的package包/类
private void waitForCounter(MetricsWALSource source, String metricName, long expectedCount)
    throws InterruptedException {
  long startWait = System.currentTimeMillis();
  long currentCount;
  while ((currentCount = metricsAssertHelper.getCounter(metricName, source)) < expectedCount) {
    Thread.sleep(100);
    if (System.currentTimeMillis() - startWait > 10000) {
      fail(String.format("Timed out waiting for '%s' >= '%s', currentCount=%s", metricName,
          expectedCount, currentCount));
    }
  }
}
 

import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; //导入依赖的package包/类
@Test
public void testBatchPutWithTsSlop() throws Exception {
  byte[] b = Bytes.toBytes(getName());
  byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
  byte[] qual = Bytes.toBytes("qual");
  byte[] val = Bytes.toBytes("val");

  // add data with a timestamp that is too recent for range. Ensure assert
  conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
  this.region = initHRegion(b, getName(), conf, cf);

  try {
    MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
    long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);

    final Put[] puts = new Put[10];
    for (int i = 0; i < 10; i++) {
      puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
      puts[i].add(cf, qual, val);
    }

    OperationStatus[] codes = this.region.batchMutate(puts);
    assertEquals(10, codes.length);
    for (int i = 0; i < 10; i++) {
      assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i].getOperationStatusCode());
    }
    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);

  } finally {
    HRegion.closeHRegion(this.region);
    this.region = null;
  }

}
 

import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; //导入依赖的package包/类
public void testBatchPutWithTsSlop() throws Exception {
  byte[] b = Bytes.toBytes(getName());
  byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
  byte[] qual = Bytes.toBytes("qual");
  byte[] val = Bytes.toBytes("val");
  Configuration conf = HBaseConfiguration.create(this.conf);

  // add data with a timestamp that is too recent for range. Ensure assert
  conf.setInt("hbase.hregion.keyvalue.timestamp.slop.millisecs", 1000);
  this.region = initHRegion(b, getName(), conf, cf);

  try{
    MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
    long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);

    final Put[] puts = new Put[10];
    for (int i = 0; i < 10; i++) {
      puts[i] = new Put(Bytes.toBytes("row_" + i), Long.MAX_VALUE - 100);
      puts[i].add(cf, qual, val);
    }

    OperationStatus[] codes = this.region.put(puts);
    assertEquals(10, codes.length);
    for (int i = 0; i < 10; i++) {
      assertEquals(OperationStatusCode.SANITY_CHECK_FAILURE, codes[i]
          .getOperationStatusCode());
    }
    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);

  } finally {
    HRegion.closeHRegion(this.region);
    this.region = null;
  }

}
 

import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; //导入依赖的package包/类
@Test
public void testBatchPut() throws Exception {
  byte[] b = Bytes.toBytes(getName());
  byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
  byte[] qual = Bytes.toBytes("qual");
  byte[] val = Bytes.toBytes("val");
  this.region = initHRegion(b, getName(), CONF, cf);
  MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
  try {
    long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);

    LOG.info("First a batch put with all valid puts");
    final Put[] puts = new Put[10];
    for (int i = 0; i < 10; i++) {
      puts[i] = new Put(Bytes.toBytes("row_" + i));
      puts[i].add(cf, qual, val);
    }

    OperationStatus[] codes = this.region.batchMutate(puts);
    assertEquals(10, codes.length);
    for (int i = 0; i < 10; i++) {
      assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
    }
    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);

    LOG.info("Next a batch put with one invalid family");
    puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
    codes = this.region.batchMutate(puts);
    assertEquals(10, codes.length);
    for (int i = 0; i < 10; i++) {
      assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
          codes[i].getOperationStatusCode());
    }

    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);

    LOG.info("Next a batch put that has to break into two batches to avoid a lock");
    RowLock rowLock = region.getRowLock(Bytes.toBytes("row_2"));

    MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
    final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
    TestThread putter = new TestThread(ctx) {
      @Override
      public void doWork() throws IOException {
        retFromThread.set(region.batchMutate(puts));
      }
    };
    LOG.info("...starting put thread while holding lock");
    ctx.addThread(putter);
    ctx.startThreads();

    LOG.info("...waiting for put thread to sync first time");
    long startWait = System.currentTimeMillis();
    while (metricsAssertHelper.getCounter("syncTimeNumOps", source) == syncs + 2) {
      Thread.sleep(100);
      if (System.currentTimeMillis() - startWait > 10000) {
        fail("Timed out waiting for thread to sync first minibatch");
      }
    }
    LOG.info("...releasing row lock, which should let put thread continue");
    rowLock.release();
    LOG.info("...joining on thread");
    ctx.stop();
    LOG.info("...checking that next batch was synced");
    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 4, source);
    codes = retFromThread.get();
    for (int i = 0; i < 10; i++) {
      assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
          codes[i].getOperationStatusCode());
    }

  } finally {
    HRegion.closeHRegion(this.region);
    this.region = null;
  }
}
 

import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; //导入依赖的package包/类
@Test
public void testAtomicBatchPut() throws IOException {
  final Put[] puts = new Put[10];
  MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
  try {
    long syncs = prepareRegionForBachPut(puts, source, false);

    // 1. Straight forward case, should succeed
    MutationBatchOperation batchOp = new MutationBatchOperation(region, puts, true,
        HConstants.NO_NONCE, HConstants.NO_NONCE);
    OperationStatus[] codes = this.region.batchMutate(batchOp);
    assertEquals(10, codes.length);
    for (int i = 0; i < 10; i++) {
      assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
    }
    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);

    // 2. Failed to get lock
    RowLock lock = region.getRowLock(Bytes.toBytes("row_" + 3));
    // Method {@link HRegion#getRowLock(byte[])} is reentrant. As 'row_3' is locked in this
    // thread, need to run {@link HRegion#batchMutate(HRegion.BatchOperation)} in different thread
    MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(CONF);
    final AtomicReference<IOException> retFromThread = new AtomicReference<>();
    final CountDownLatch finishedPuts = new CountDownLatch(1);
    final MutationBatchOperation finalBatchOp = new MutationBatchOperation(region, puts, true,
        HConstants
        .NO_NONCE,
        HConstants.NO_NONCE);
    TestThread putter = new TestThread(ctx) {
      @Override
      public void doWork() throws IOException {
        try {
          region.batchMutate(finalBatchOp);
        } catch (IOException ioe) {
          LOG.error("test failed!", ioe);
          retFromThread.set(ioe);
        }
        finishedPuts.countDown();
      }
    };
    LOG.info("...starting put thread while holding locks");
    ctx.addThread(putter);
    ctx.startThreads();
    LOG.info("...waiting for batch puts while holding locks");
    try {
      finishedPuts.await();
    } catch (InterruptedException e) {
      LOG.error("Interrupted!", e);
    } finally {
      if (lock != null) {
        lock.release();
      }
    }
    assertNotNull(retFromThread.get());
    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);

    // 3. Exception thrown in validation
    LOG.info("Next a batch put with one invalid family");
    puts[5].addColumn(Bytes.toBytes("BAD_CF"), qual, value);
    batchOp = new MutationBatchOperation(region, puts, true, HConstants.NO_NONCE,
        HConstants.NO_NONCE);
    thrown.expect(NoSuchColumnFamilyException.class);
    this.region.batchMutate(batchOp);
  } finally {
    HBaseTestingUtility.closeRegionAndWAL(this.region);
    this.region = null;
  }
}
 

import org.apache.hadoop.hbase.regionserver.wal.MetricsWALSource; //导入依赖的package包/类
@Test
public void testBatchPut() throws Exception {
  byte[] b = Bytes.toBytes(getName());
  byte[] cf = Bytes.toBytes(COLUMN_FAMILY);
  byte[] qual = Bytes.toBytes("qual");
  byte[] val = Bytes.toBytes("val");
  this.region = initHRegion(b, getName(), conf, cf);
  MetricsWALSource source = CompatibilitySingletonFactory.getInstance(MetricsWALSource.class);
  try {
    long syncs = metricsAssertHelper.getCounter("syncTimeNumOps", source);
    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs, source);

    LOG.info("First a batch put with all valid puts");
    final Put[] puts = new Put[10];
    for (int i = 0; i < 10; i++) {
      puts[i] = new Put(Bytes.toBytes("row_" + i));
      puts[i].add(cf, qual, val);
    }

    OperationStatus[] codes = this.region.batchMutate(puts);
    assertEquals(10, codes.length);
    for (int i = 0; i < 10; i++) {
      assertEquals(OperationStatusCode.SUCCESS, codes[i].getOperationStatusCode());
    }
    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 1, source);

    LOG.info("Next a batch put with one invalid family");
    puts[5].add(Bytes.toBytes("BAD_CF"), qual, val);
    codes = this.region.batchMutate(puts);
    assertEquals(10, codes.length);
    for (int i = 0; i < 10; i++) {
      assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
          codes[i].getOperationStatusCode());
    }

    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 2, source);

    LOG.info("Next a batch put that has to break into two batches to avoid a lock");
    RowLock rowLock = region.getRowLock(Bytes.toBytes("row_2"));

    MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(conf);
    final AtomicReference<OperationStatus[]> retFromThread = new AtomicReference<OperationStatus[]>();
    TestThread putter = new TestThread(ctx) {
      @Override
      public void doWork() throws IOException {
        retFromThread.set(region.batchMutate(puts));
      }
    };
    LOG.info("...starting put thread while holding lock");
    ctx.addThread(putter);
    ctx.startThreads();

    LOG.info("...waiting for put thread to sync first time");
    long startWait = System.currentTimeMillis();
    while (metricsAssertHelper.getCounter("syncTimeNumOps", source) == syncs + 2) {
      Thread.sleep(100);
      if (System.currentTimeMillis() - startWait > 10000) {
        fail("Timed out waiting for thread to sync first minibatch");
      }
    }
    LOG.info("...releasing row lock, which should let put thread continue");
    rowLock.release();
    LOG.info("...joining on thread");
    ctx.stop();
    LOG.info("...checking that next batch was synced");
    metricsAssertHelper.assertCounter("syncTimeNumOps", syncs + 4, source);
    codes = retFromThread.get();
    for (int i = 0; i < 10; i++) {
      assertEquals((i == 5) ? OperationStatusCode.BAD_FAMILY : OperationStatusCode.SUCCESS,
          codes[i].getOperationStatusCode());
    }

  } finally {
    HRegion.closeHRegion(this.region);
    this.region = null;
  }
}