Java HFile.createReader方法代码示例

import org.apache.hadoop.hbase.io.hfile.HFile; //导入方法依赖的package包/类
/**
 * Verify that compression and data block encoding are respected by the
 * Store.createWriterInTmp() method, used on store flush.
 */
@Test
public void testCreateWriter() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  FileSystem fs = FileSystem.get(conf);

  HColumnDescriptor hcd = new HColumnDescriptor(family);
  hcd.setCompressionType(Compression.Algorithm.GZ);
  hcd.setDataBlockEncoding(DataBlockEncoding.DIFF);
  init(name.getMethodName(), conf, hcd);

  // Test createWriterInTmp()
  StoreFile.Writer writer = store.createWriterInTmp(4, hcd.getCompression(), false, true, false);
  Path path = writer.getPath();
  writer.append(new KeyValue(row, family, qf1, Bytes.toBytes(1)));
  writer.append(new KeyValue(row, family, qf2, Bytes.toBytes(2)));
  writer.append(new KeyValue(row2, family, qf1, Bytes.toBytes(3)));
  writer.append(new KeyValue(row2, family, qf2, Bytes.toBytes(4)));
  writer.close();

  // Verify that compression and encoding settings are respected
  HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), conf);
  Assert.assertEquals(hcd.getCompressionType(), reader.getCompressionAlgorithm());
  Assert.assertEquals(hcd.getDataBlockEncoding(), reader.getDataBlockEncoding());
  reader.close();
}
 

import org.apache.hadoop.hbase.io.hfile.HFile; //导入方法依赖的package包/类
private static byte[] extractHFileKey(Path path) throws Exception {
  HFile.Reader reader = HFile.createReader(TEST_UTIL.getTestFileSystem(), path,
    new CacheConfig(conf), conf);
  try {
    reader.loadFileInfo();
    Encryption.Context cryptoContext = reader.getFileContext().getEncryptionContext();
    assertNotNull("Reader has a null crypto context", cryptoContext);
    Key key = cryptoContext.getKey();
    if (key == null) {
      return null;
    }
    return key.getEncoded();
  } finally {
    reader.close();
  }
}
 

import org.apache.hadoop.hbase.io.hfile.HFile; //导入方法依赖的package包/类
public static void doSmokeTest(FileSystem fs, Path path, String codec)
throws Exception {
  Configuration conf = HBaseConfiguration.create();
  HFile.Writer writer = HFile.getWriterFactoryNoCache(conf)
      .withPath(fs, path)
      .withCompression(codec)
      .create();
  writer.append(Bytes.toBytes("testkey"), Bytes.toBytes("testval"));
  writer.appendFileInfo(Bytes.toBytes("infokey"), Bytes.toBytes("infoval"));
  writer.close();

  HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf));
  reader.loadFileInfo();
  byte[] key = reader.getFirstKey();
  boolean rc = Bytes.toString(key).equals("testkey");
  reader.close();

  if (!rc) {
    throw new Exception("Read back incorrect result: " +
                        Bytes.toStringBinary(key));
  }
}
 

import org.apache.hadoop.hbase.io.hfile.HFile; //导入方法依赖的package包/类
private void readHFileSeqId(Path file, Compression.Algorithm compression) throws Exception
{
  CacheConfig cacheConf = new CacheConfig(conf);
  HFile.Reader reader = HFile.createReader(hdfs, file, cacheConf, conf);
  HFileScanner scanner = reader.getScanner(true, true, false);

  @SuppressWarnings("unused")
  KeyValue kv = null;
  scanner.seekTo();

  for (int i = 0; i < testSize; i++) {
    scanner.seekTo(getKey(i).getBytes());
    kv = scanner.getKeyValue();
    //logger.debug("key: {} value: {}", new String (kv.getKey()), new String (kv.getValue()));
  }
}
 

import org.apache.hadoop.hbase.io.hfile.HFile; //导入方法依赖的package包/类
private void readHFileRandom(Path file, Compression.Algorithm compression) throws Exception
{
  CacheConfig cacheConf = new CacheConfig(conf);
  HFile.Reader reader = HFile.createReader(hdfs, file, cacheConf, conf);
  HFileScanner scanner = reader.getScanner(true, true, false);

  @SuppressWarnings("unused")
  KeyValue kv = null;
  scanner.seekTo();
  Random random = new Random();
  for (int i = 0; i < testSize; i++) {
    scanner.seekTo();
    scanner.seekTo(getKey(random.nextInt(testSize)).getBytes());
    kv = scanner.getKeyValue();
    //logger.debug("key: {} value: {}", new String (kv.getKey()), new String (kv.getValue()));
  }
}
 

import org.apache.hadoop.hbase.io.hfile.HFile; //导入方法依赖的package包/类
private void scanHFileOnLocalFS(Path file) throws IOException {
  HColumnDescriptor family = desc.getFamily(Bytes.toBytes("f"));
  CacheConfig cacheConf = new CacheConfig(conf, family);
  HFile.Reader reader = HFile.createReader(LocalFileSystem.getLocal(conf), file, cacheConf, conf);
  HFileScanner scanner = reader.getScanner(false, false, false);
  scanner.seekTo();
  int n = 0;
  do {
    Cell cell = scanner.getKeyValue();
    printKV(cell);
    ++n;
  } while (scanner.next());
}
 

import org.apache.hadoop.hbase.io.hfile.HFile; //导入方法依赖的package包/类
public static void doSmokeTest(FileSystem fs, Path path, String codec)
throws Exception {
  Configuration conf = HBaseConfiguration.create();
  HFileContext context = new HFileContextBuilder()
                         .withCompression(AbstractHFileWriter.compressionByName(codec)).build();
  HFile.Writer writer = HFile.getWriterFactoryNoCache(conf)
      .withPath(fs, path)
      .withFileContext(context)
      .create();
  // Write any-old Cell...
  final byte [] rowKey = Bytes.toBytes("compressiontestkey");
  Cell c = CellUtil.createCell(rowKey, Bytes.toBytes("compressiontestval"));
  writer.append(c);
  writer.appendFileInfo(Bytes.toBytes("compressioninfokey"), Bytes.toBytes("compressioninfoval"));
  writer.close();
  Cell cc = null;
  HFile.Reader reader = HFile.createReader(fs, path, new CacheConfig(conf), conf);
  try {
    reader.loadFileInfo();
    HFileScanner scanner = reader.getScanner(false, true);
    scanner.seekTo(); // position to the start of file
    // Scanner does not do Cells yet. Do below for now till fixed.
    cc = scanner.getKeyValue();
    if (CellComparator.compareRows(c, cc) != 0) {
      throw new Exception("Read back incorrect result: " + c.toString() + " vs " + cc.toString());
    }
  } finally {
    reader.close();
  }
}
 

import org.apache.hadoop.hbase.io.hfile.HFile; //导入方法依赖的package包/类
/**
 * Checks a path to see if it is a valid hfile.
 *
 * @param p
 *          full Path to an HFile
 * @throws IOException
 *           This is a connectivity related exception
 */
protected void checkHFile(Path p) throws IOException {
  HFile.Reader r = null;
  try {
    r = HFile.createReader(fs, p, cacheConf, conf);
  } catch (CorruptHFileException che) {
    LOG.warn("Found corrupt HFile " + p, che);
    corrupted.add(p);
    if (inQuarantineMode) {
      Path dest = createQuarantinePath(p);
      LOG.warn("Quarantining corrupt HFile " + p + " into " + dest);
      boolean success = fs.mkdirs(dest.getParent());
      success = success ? fs.rename(p, dest): false;
      if (!success) {
        failures.add(p);
      } else {
        quarantined.add(dest);
      }
    }
    return;
  } catch (FileNotFoundException fnfe) {
    LOG.warn("HFile " + p + " was missing.  Likely removed due to compaction/split?");
    missing.add(p);
  } finally {
    hfilesChecked.addAndGet(1);
    if (r != null) {
      r.close(true);
    }
  }
}
 

import org.apache.hadoop.hbase.io.hfile.HFile; //导入方法依赖的package包/类
/**
 * Do a minor/major compaction for store files' index.
 *
 * @param compactedFile if null, compact index from this file, else compact each StoreFile's index
 *                      together
 * @return Product of compaction or null if there is no index column cell
 * @throws IOException
 */
void compactLMDIndex(final Path compactedFile, HStore store, TimeRangeTracker timeRangeTracker)
    throws IOException {
  if (LOG.isDebugEnabled()) {
    LOG.debug(
        "Generate intermediate index file from major compaction file=" + compactedFile + " in cf="
            + store.toString());
  }
  HFile.Reader reader =
      HFile.createReader(store.fs.getFileSystem(), compactedFile, store.cacheConf, conf);
  HFileScanner scanner = reader.getScanner(false, false, true);
  Queue<KeyValue> rawRecords = new LinkedList<>();
  int counter = 0;
  try {
    scanner.seekTo();
    do {
      KeyValue kv = (KeyValue) scanner.getKeyValue();
      if (store.indexTableRelation.isIndexColumn(kv.getFamily(), kv.getQualifier())) {
        rawRecords.add(kv);
      }
      ++counter;
    } while (scanner.next());
  } finally {
    if (reader != null) reader.close();
  }
  System.out.println("in compacted file=" + compactedFile + ", number of keyvalue=" + counter
      + ", for LMDIndex is:" + rawRecords.size());
  LMDIndexWriter lmdIndexWriter =
      new LMDIndexWriter(store, compactedFile, timeRangeTracker, "COMPACT");
  lmdIndexWriter.processKeyValueQueue(rawRecords);
}
 

import org.apache.hadoop.hbase.io.hfile.HFile; //导入方法依赖的package包/类
private Queue<KeyValue> parseKeyValueFromReader() throws IOException {
  HFile.Reader reader = HFile
      .createReader(store.getFileSystem(), majorMovedPath, store.getCacheConfig(), store.conf);
  HFileScanner scanner = reader.getScanner(false, false, false);
  scanner.seekTo();
  Queue<KeyValue> queue = new LinkedList<>();
  do {
    KeyValue cell = (KeyValue) scanner.getKeyValue();
    queue.offer(cell);
  } while (scanner.next());
  reader.close();
  return queue;
}
 

import org.apache.hadoop.hbase.io.hfile.HFile; //导入方法依赖的package包/类
private void winterTestingHFile(Path file) throws IOException {
  HFile.Reader reader =
      HFile.createReader(store.getFileSystem(), file, store.getCacheConfig(), store.conf);
  HFileScanner scanner = reader.getScanner(false, false, false);
  scanner.seekTo();
  int n = 0;
  do {
    Cell cell = scanner.getKeyValue();
    ++n;
  } while (scanner.next());
  LOG.info("LCDBG, HFile has: " + n + " in " + file);
}
 

import org.apache.hadoop.hbase.io.hfile.HFile; //导入方法依赖的package包/类
/**
 * Method returns the total KVs in given hfile
 * @param fs File System
 * @param p HFile path
 * @return KV count in the given hfile
 * @throws IOException
 */
private static int getKVCountFromHfile(FileSystem fs, Path p) throws IOException {
  Configuration conf = util.getConfiguration();
  HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);
  reader.loadFileInfo();
  HFileScanner scanner = reader.getScanner(false, false);
  scanner.seekTo();
  int count = 0;
  do {
    count++;
  } while (scanner.next());
  reader.close();
  return count;
}
 

import org.apache.hadoop.hbase.io.hfile.HFile; //导入方法依赖的package包/类
private int verifyHFile(Path p) throws IOException {
  Configuration conf = util.getConfiguration();
  HFile.Reader reader = HFile.createReader(
      p.getFileSystem(conf), p, new CacheConfig(conf), conf);
  reader.loadFileInfo();
  HFileScanner scanner = reader.getScanner(false, false);
  scanner.seekTo();
  int count = 0;
  do {
    count++;
  } while (scanner.next());
  assertTrue(count > 0);
  reader.close();
  return count;
}
 

import org.apache.hadoop.hbase.io.hfile.HFile; //导入方法依赖的package包/类
private byte[] extractHFileKey(Path path) throws Exception {
  HFile.Reader reader = HFile.createReader(TEST_UTIL.getTestFileSystem(), path,
    new CacheConfig(conf), conf);
  try {
    reader.loadFileInfo();
    Encryption.Context cryptoContext = reader.getFileContext().getEncryptionContext();
    assertNotNull("Reader has a null crypto context", cryptoContext);
    Key key = cryptoContext.getKey();
    assertNotNull("Crypto context has no key", key);
    return key.getEncoded();
  } finally {
    reader.close();
  }
}
 

import org.apache.hadoop.hbase.io.hfile.HFile; //导入方法依赖的package包/类
private static byte[] extractHFileKey(Path path) throws Exception {
  HFile.Reader reader = HFile.createReader(TEST_UTIL.getTestFileSystem(), path,
    new CacheConfig(conf), conf);
  try {
    reader.loadFileInfo();
    Encryption.Context cryptoContext = reader.getFileContext().getEncryptionContext();
    assertNotNull("Reader has a null crypto context", cryptoContext);
    Key key = cryptoContext.getKey();
    assertNotNull("Crypto context has no key", key);
    return key.getEncoded();
  } finally {
    reader.close();
  }
}