Java KeyValue.COMPARATOR屬性代碼示例

public AbstractHFileWriter(CacheConfig cacheConf,
    FSDataOutputStream outputStream, Path path, 
    KVComparator comparator, HFileContext fileContext) {
  this.outputStream = outputStream;
  this.path = path;
  this.name = path != null ? path.getName() : outputStream.toString();
  this.hFileContext = fileContext;
  DataBlockEncoding encoding = hFileContext.getDataBlockEncoding();
  if (encoding != DataBlockEncoding.NONE) {
    this.blockEncoder = new HFileDataBlockEncoderImpl(encoding);
  } else {
    this.blockEncoder = NoOpDataBlockEncoder.INSTANCE;
  }
  this.comparator = comparator != null ? comparator
      : KeyValue.COMPARATOR;

  closeOutputStream = path != null;
  this.cacheConf = cacheConf;
}
 

public void testMultipleVersionsSimple() throws Exception {
  DefaultMemStore m = new DefaultMemStore(new Configuration(), KeyValue.COMPARATOR);
  byte [] row = Bytes.toBytes("testRow");
  byte [] family = Bytes.toBytes("testFamily");
  byte [] qf = Bytes.toBytes("testQualifier");
  long [] stamps = {1,2,3};
  byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"),
      Bytes.toBytes("value2")};
  KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]);
  KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]);
  KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]);

  m.add(key0);
  m.add(key1);
  m.add(key2);

  assertTrue("Expected memstore to hold 3 values, actually has " +
      m.cellSet.size(), m.cellSet.size() == 3);
}
 

public void setUp() throws Exception {
  super.setUp();
  row1 = Bytes.toBytes("row1");
  row2 = Bytes.toBytes("row2");
  row3 = Bytes.toBytes("row3");
  fam1 = Bytes.toBytes("fam1");
  fam2 = Bytes.toBytes("fam2");
  col1 = Bytes.toBytes("col1");
  col2 = Bytes.toBytes("col2");
  col3 = Bytes.toBytes("col3");
  col4 = Bytes.toBytes("col4");
  col5 = Bytes.toBytes("col5");

  data = Bytes.toBytes("data");

  //Create Get
  get = new Get(row1);
  get.addFamily(fam1);
  get.addColumn(fam2, col2);
  get.addColumn(fam2, col4);
  get.addColumn(fam2, col5);
  this.scan = new Scan(get);

  rowComparator = KeyValue.COMPARATOR;

}
 

/**
 * Creates a new general (Row or RowCol) Bloom filter at the time of
 * {@link org.apache.hadoop.hbase.regionserver.StoreFile} writing.
 *
 * @param conf
 * @param cacheConf
 * @param bloomType
 * @param maxKeys an estimate of the number of keys we expect to insert.
 *        Irrelevant if compound Bloom filters are enabled.
 * @param writer the HFile writer
 * @return the new Bloom filter, or null in case Bloom filters are disabled
 *         or when failed to create one.
 */
public static BloomFilterWriter createGeneralBloomAtWrite(Configuration conf,
    CacheConfig cacheConf, BloomType bloomType, int maxKeys,
    HFile.Writer writer) {
  if (!isGeneralBloomEnabled(conf)) {
    LOG.trace("Bloom filters are disabled by configuration for "
        + writer.getPath()
        + (conf == null ? " (configuration is null)" : ""));
    return null;
  } else if (bloomType == BloomType.NONE) {
    LOG.trace("Bloom filter is turned off for the column family");
    return null;
  }

  float err = getErrorRate(conf);

  // In case of row/column Bloom filter lookups, each lookup is an OR if two
  // separate lookups. Therefore, if each lookup's false positive rate is p,
  // the resulting false positive rate is err = 1 - (1 - p)^2, and
  // p = 1 - sqrt(1 - err).
  if (bloomType == BloomType.ROWCOL) {
    err = (float) (1 - Math.sqrt(1 - err));
  }

  int maxFold = conf.getInt(IO_STOREFILE_BLOOM_MAX_FOLD,
      MAX_ALLOWED_FOLD_FACTOR);

  // Do we support compound bloom filters?
  // In case of compound Bloom filters we ignore the maxKeys hint.
  CompoundBloomFilterWriter bloomWriter = new CompoundBloomFilterWriter(getBloomBlockSize(conf),
      err, Hash.getHashType(conf), maxFold, cacheConf.shouldCacheBloomsOnWrite(),
      bloomType == BloomType.ROWCOL ? KeyValue.COMPARATOR : KeyValue.RAW_COMPARATOR);
  writer.addInlineBlockWriter(bloomWriter);
  return bloomWriter;
}
 

/**
 * Create a store file writer. Client is responsible for closing file when done. If metadata,
 * add BEFORE closing using {@link Writer#appendMetadata}.
 */
public Writer build() throws IOException {
  if ((dir == null ? 0 : 1) + (filePath == null ? 0 : 1) != 1) {
    throw new IllegalArgumentException("Either specify parent directory " + "or file path");
  }

  if (dir == null) {
    dir = filePath.getParent();
  }

  if (!fs.exists(dir)) {
    fs.mkdirs(dir);
  }

  if (filePath == null) {
    filePath = getUniqueFile(fs, dir);
    if (!BloomFilterFactory.isGeneralBloomEnabled(conf)) {
      bloomType = BloomType.NONE;
    }
  }

  if (comparator == null) {
    comparator = KeyValue.COMPARATOR;
  }
  return new Writer(fs, filePath, conf, cacheConf, comparator, bloomType, maxKeyCount,
      favoredNodes, fileContext);
}
 

/**
 * Test a pathological pattern that shows why we can't currently
 * use the MSLAB for upsert workloads. This test inserts data
 * in the following pattern:
 *
 * - row0001 through row1000 (fills up one 2M Chunk)
 * - row0002 through row1001 (fills up another 2M chunk, leaves one reference
 *   to the first chunk
 * - row0003 through row1002 (another chunk, another dangling reference)
 *
 * This causes OOME pretty quickly if we use MSLAB for upsert
 * since each 2M chunk is held onto by a single reference.
 */
public void testUpsertMSLAB() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  conf.setBoolean(DefaultMemStore.USEMSLAB_KEY, true);
  memstore = new DefaultMemStore(conf, KeyValue.COMPARATOR);

  int ROW_SIZE = 2048;
  byte[] qualifier = new byte[ROW_SIZE - 4];

  MemoryMXBean bean = ManagementFactory.getMemoryMXBean();
  for (int i = 0; i < 3; i++) { System.gc(); }
  long usageBefore = bean.getHeapMemoryUsage().getUsed();

  long size = 0;
  long ts=0;

  for (int newValue = 0; newValue < 1000; newValue++) {
    for (int row = newValue; row < newValue + 1000; row++) {
      byte[] rowBytes = Bytes.toBytes(row);
      size += memstore.updateColumnValue(rowBytes, FAMILY, qualifier, newValue, ++ts);
    }
  }
  System.out.println("Wrote " + ts + " vals");
  for (int i = 0; i < 3; i++) { System.gc(); }
  long usageAfter = bean.getHeapMemoryUsage().getUsed();
  System.out.println("Memory used: " + (usageAfter - usageBefore)
      + " (heapsize: " + memstore.heapSize() +
      " size: " + size + ")");
}
 

/**
 * Add keyvalues with a fixed memstoreTs, and checks that memstore size is decreased
 * as older keyvalues are deleted from the memstore.
 * @throws Exception
 */
public void testUpsertMemstoreSize() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  memstore = new DefaultMemStore(conf, KeyValue.COMPARATOR);
  long oldSize = memstore.size.get();

  List<Cell> l = new ArrayList<Cell>();
  KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
  KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v");
  KeyValue kv3 = KeyValueTestUtil.create("r", "f", "q", 102, "v");

  kv1.setSequenceId(1); kv2.setSequenceId(1);kv3.setSequenceId(1);
  l.add(kv1); l.add(kv2); l.add(kv3);

  this.memstore.upsert(l, 2);// readpoint is 2
  long newSize = this.memstore.size.get();
  assert(newSize > oldSize);
  //The kv1 should be removed.
  assert(memstore.cellSet.size() == 2);
  
  KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v");
  kv4.setSequenceId(1);
  l.clear(); l.add(kv4);
  this.memstore.upsert(l, 3);
  assertEquals(newSize, this.memstore.size.get());
  //The kv2 should be removed.
  assert(memstore.cellSet.size() == 2);
  //this.memstore = null;
}
 

public void testScannerLeak() throws IOException {
  // Test for unclosed scanners (HBASE-1927)

  List<Cell> l1 = new ArrayList<Cell>();
  l1.add(new KeyValue(row1, fam1, col5, data));
  l1.add(new KeyValue(row2, fam1, col1, data));
  l1.add(new KeyValue(row2, fam1, col2, data));
  scanners.add(new Scanner(l1));

  List<Cell> l2 = new ArrayList<Cell>();
  l2.add(new KeyValue(row1, fam1, col1, data));
  l2.add(new KeyValue(row1, fam1, col2, data));
  scanners.add(new Scanner(l2));

  List<Cell> l3 = new ArrayList<Cell>();
  l3.add(new KeyValue(row1, fam1, col3, data));
  l3.add(new KeyValue(row1, fam1, col4, data));
  l3.add(new KeyValue(row1, fam2, col1, data));
  l3.add(new KeyValue(row1, fam2, col2, data));
  l3.add(new KeyValue(row2, fam1, col3, data));
  scanners.add(new Scanner(l3));

  List<Cell> l4 = new ArrayList<Cell>();
  scanners.add(new Scanner(l4));

  //Creating KeyValueHeap
  KeyValueHeap kvh = new KeyValueHeap(scanners, KeyValue.COMPARATOR);

  while(kvh.next() != null);

  for(KeyValueScanner scanner : scanners) {
    assertTrue(((Scanner)scanner).isClosed());
  }
}
 

public void testKeyValueScanFixture() throws IOException {
  KeyValue kvs[] = new KeyValue[]{
      KeyValueTestUtil.create("RowA", "family", "qf1",
          1, KeyValue.Type.Put, "value-1"),
      KeyValueTestUtil.create("RowA", "family", "qf2",
          1, KeyValue.Type.Put, "value-2"),
      KeyValueTestUtil.create("RowB", "family", "qf1",
          10, KeyValue.Type.Put, "value-10")
  };
  KeyValueScanner scan = new KeyValueScanFixture(
      KeyValue.COMPARATOR, kvs);

  KeyValue kv = KeyValueUtil.createFirstOnRow(Bytes.toBytes("RowA"));
  // should seek to this:
  assertTrue(scan.seek(kv));
  Cell res = scan.peek();
  assertEquals(kvs[0], res);

  kv = KeyValueUtil.createFirstOnRow(Bytes.toBytes("RowB"));
  assertTrue(scan.seek(kv));
  res = scan.peek();
  assertEquals(kvs[2], res);

  // ensure we pull things out properly:
  kv = KeyValueUtil.createFirstOnRow(Bytes.toBytes("RowA"));
  assertTrue(scan.seek(kv));
  assertEquals(kvs[0], scan.peek());
  assertEquals(kvs[0], scan.next());
  assertEquals(kvs[1], scan.peek());
  assertEquals(kvs[1], scan.next());
  assertEquals(kvs[2], scan.peek());
  assertEquals(kvs[2], scan.next());
  assertEquals(null, scan.peek());
  assertEquals(null, scan.next());
}
 

public void testWildCardTtlScan() throws IOException {
  long now = System.currentTimeMillis();
  KeyValue [] kvs = new KeyValue[] {
      KeyValueTestUtil.create("R1", "cf", "a", now-1000, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R1", "cf", "d", now-10000, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R2", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R2", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R2", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
      KeyValueTestUtil.create("R2", "cf", "c", now-1000, KeyValue.Type.Put, "dont-care")
  };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  Scan scan = new Scan();
  scan.setMaxVersions(1);
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE, 0,
      KeyValue.COMPARATOR);
  ScanType scanType = ScanType.USER_SCAN;
  StoreScanner scanner =
    new StoreScanner(scan, scanInfo, scanType,
        null, scanners);

  List<Cell> results = new ArrayList<Cell>();
  assertEquals(true, scanner.next(results));
  assertEquals(2, results.size());
  assertEquals(kvs[1], results.get(0));
  assertEquals(kvs[2], results.get(1));
  results.clear();

  assertEquals(true, scanner.next(results));
  assertEquals(3, results.size());
  assertEquals(kvs[4], results.get(0));
  assertEquals(kvs[5], results.get(1));
  assertEquals(kvs[6], results.get(2));
  results.clear();

  assertEquals(false, scanner.next(results));
}
 

/**
 * Ensure that expired delete family markers don't override valid puts
 */
public void testExpiredDeleteFamily() throws Exception {
  long now = System.currentTimeMillis();
  KeyValue [] kvs = new KeyValue[] {
      new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now-1000,
          KeyValue.Type.DeleteFamily),
      KeyValueTestUtil.create("R1", "cf", "a", now-10, KeyValue.Type.Put,
          "dont-care"),
  };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  Scan scan = new Scan();
  scan.setMaxVersions(1);
  // scanner with ttl equal to 500
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE, 0,
      KeyValue.COMPARATOR);
  ScanType scanType = ScanType.USER_SCAN;
  StoreScanner scanner =
      new StoreScanner(scan, scanInfo, scanType, null, scanners);

  List<Cell> results = new ArrayList<Cell>();
  assertEquals(true, scanner.next(results));
  assertEquals(1, results.size());
  assertEquals(kvs[1], results.get(0));
  results.clear();

  assertEquals(false, scanner.next(results));
}
 

public CollectionBackedScanner(SortedSet<Cell> set) {
  this(set, KeyValue.COMPARATOR);
}
 

/**
 * Default constructor. Used for tests.
 */
public DefaultMemStore() {
  this(HBaseConfiguration.create(), KeyValue.COMPARATOR);
}
 

public void testSorted() throws IOException{
  //Cases that need to be checked are:
  //1. The "smallest" KeyValue is in the same scanners as current
  //2. Current scanner gets empty

  List<Cell> l1 = new ArrayList<Cell>();
  l1.add(new KeyValue(row1, fam1, col5, data));
  l1.add(new KeyValue(row2, fam1, col1, data));
  l1.add(new KeyValue(row2, fam1, col2, data));
  scanners.add(new Scanner(l1));

  List<Cell> l2 = new ArrayList<Cell>();
  l2.add(new KeyValue(row1, fam1, col1, data));
  l2.add(new KeyValue(row1, fam1, col2, data));
  scanners.add(new Scanner(l2));

  List<Cell> l3 = new ArrayList<Cell>();
  l3.add(new KeyValue(row1, fam1, col3, data));
  l3.add(new KeyValue(row1, fam1, col4, data));
  l3.add(new KeyValue(row1, fam2, col1, data));
  l3.add(new KeyValue(row1, fam2, col2, data));
  l3.add(new KeyValue(row2, fam1, col3, data));
  scanners.add(new Scanner(l3));

  List<KeyValue> expected = new ArrayList<KeyValue>();
  expected.add(new KeyValue(row1, fam1, col1, data));
  expected.add(new KeyValue(row1, fam1, col2, data));
  expected.add(new KeyValue(row1, fam1, col3, data));
  expected.add(new KeyValue(row1, fam1, col4, data));
  expected.add(new KeyValue(row1, fam1, col5, data));
  expected.add(new KeyValue(row1, fam2, col1, data));
  expected.add(new KeyValue(row1, fam2, col2, data));
  expected.add(new KeyValue(row2, fam1, col1, data));
  expected.add(new KeyValue(row2, fam1, col2, data));
  expected.add(new KeyValue(row2, fam1, col3, data));

  //Creating KeyValueHeap
  KeyValueHeap kvh =
    new KeyValueHeap(scanners, KeyValue.COMPARATOR);

  List<Cell> actual = new ArrayList<Cell>();
  while(kvh.peek() != null){
    actual.add(kvh.next());
  }

  assertEquals(expected.size(), actual.size());
  for(int i=0; i<expected.size(); i++){
    assertEquals(expected.get(i), actual.get(i));
    if(PRINT){
      System.out.println("expected " +expected.get(i)+
          "\nactual   " +actual.get(i) +"\n");
    }
  }

  //Check if result is sorted according to Comparator
  for(int i=0; i<actual.size()-1; i++){
    int ret = KeyValue.COMPARATOR.compare(actual.get(i), actual.get(i+1));
    assertTrue(ret < 0);
  }

}
 

public void testSeek() throws IOException {
  //Cases:
  //1. Seek KeyValue that is not in scanner
  //2. Check that smallest that is returned from a seek is correct

  List<Cell> l1 = new ArrayList<Cell>();
  l1.add(new KeyValue(row1, fam1, col5, data));
  l1.add(new KeyValue(row2, fam1, col1, data));
  l1.add(new KeyValue(row2, fam1, col2, data));
  scanners.add(new Scanner(l1));

  List<Cell> l2 = new ArrayList<Cell>();
  l2.add(new KeyValue(row1, fam1, col1, data));
  l2.add(new KeyValue(row1, fam1, col2, data));
  scanners.add(new Scanner(l2));

  List<Cell> l3 = new ArrayList<Cell>();
  l3.add(new KeyValue(row1, fam1, col3, data));
  l3.add(new KeyValue(row1, fam1, col4, data));
  l3.add(new KeyValue(row1, fam2, col1, data));
  l3.add(new KeyValue(row1, fam2, col2, data));
  l3.add(new KeyValue(row2, fam1, col3, data));
  scanners.add(new Scanner(l3));

  List<KeyValue> expected = new ArrayList<KeyValue>();
  expected.add(new KeyValue(row2, fam1, col1, data));

  //Creating KeyValueHeap
  KeyValueHeap kvh =
    new KeyValueHeap(scanners, KeyValue.COMPARATOR);

  KeyValue seekKv = new KeyValue(row2, fam1, null, null);
  kvh.seek(seekKv);

  List<Cell> actual = new ArrayList<Cell>();
  actual.add(kvh.peek());

  assertEquals(expected.size(), actual.size());
  for(int i=0; i<expected.size(); i++){
    assertEquals(expected.get(i), actual.get(i));
    if(PRINT){
      System.out.println("expected " +expected.get(i)+
          "\nactual   " +actual.get(i) +"\n");
    }
  }

}