Java BloomFilterFactory.createDeleteBloomAtWrite方法代码示例

import org.apache.hadoop.hbase.util.BloomFilterFactory; //导入方法依赖的package包/类
/**
 * Creates an HFile.Writer that also write helpful meta data.
 *
 * @param fs           file system to write to
 * @param path         file name to create
 * @param conf         user configuration
 * @param comparator   key comparator
 * @param bloomType    bloom filter setting
 * @param maxKeys      the expected maximum number of keys to be added. Was used for Bloom filter
 *                     size in {@link HFile} format version 1.
 * @param favoredNodes
 * @param fileContext  - The HFile context
 * @throws IOException problem writing to FS
 */
private Writer(FileSystem fs, Path path, final Configuration conf, CacheConfig cacheConf,
    final KVComparator comparator, BloomType bloomType, long maxKeys,
    InetSocketAddress[] favoredNodes, HFileContext fileContext) throws IOException {
  writer = HFile.getWriterFactory(conf, cacheConf).withPath(fs, path).withComparator(comparator)
      .withFavoredNodes(favoredNodes).withFileContext(fileContext).create();

  this.kvComparator = comparator;

  generalBloomFilterWriter = BloomFilterFactory
      .createGeneralBloomAtWrite(conf, cacheConf, bloomType,
          (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);

  if (generalBloomFilterWriter != null) {
    this.bloomType = bloomType;
    if (LOG.isTraceEnabled()) LOG.trace(
        "Bloom filter type for " + path + ": " + this.bloomType + ", "
            + generalBloomFilterWriter.getClass().getSimpleName());
  } else {
    // Not using Bloom filters.
    this.bloomType = BloomType.NONE;
  }

  // initialize delete family Bloom filter when there is NO RowCol Bloom
  // filter
  if (this.bloomType != BloomType.ROWCOL) {
    this.deleteFamilyBloomFilterWriter = BloomFilterFactory
        .createDeleteBloomAtWrite(conf, cacheConf, (int) Math.min(maxKeys, Integer.MAX_VALUE),
            writer);
  } else {
    deleteFamilyBloomFilterWriter = null;
  }
  if (deleteFamilyBloomFilterWriter != null) {
    if (LOG.isTraceEnabled()) LOG.trace(
        "Delete Family Bloom filter type for " + path + ": " + deleteFamilyBloomFilterWriter
            .getClass().getSimpleName());
  }
}
 

import org.apache.hadoop.hbase.util.BloomFilterFactory; //导入方法依赖的package包/类
/**
 * Creates an HFile.Writer that also write helpful meta data.
 * @param fs file system to write to
 * @param path file name to create
 * @param blocksize HDFS block size
 * @param compress HDFS block compression
 * @param conf user configuration
 * @param comparator key comparator
 * @param bloomType bloom filter setting
 * @param maxKeys the expected maximum number of keys to be added. Was used for Bloom filter
 *          size in {@link HFile} format version 1.
 * @param checksumType the checksum type
 * @param bytesPerChecksum the number of bytes per checksum value
 * @param includeMVCCReadpoint whether to write the mvcc readpoint to the file for each KV
 * @throws IOException problem writing to FS
 */
private Writer(FileSystem fs, Path path, int blocksize, Compression.Algorithm compress,
    HFileDataBlockEncoder dataBlockEncoder, final Configuration conf, CacheConfig cacheConf,
    final KVComparator comparator, BloomType bloomType, long maxKeys,
    final ChecksumType checksumType, final int bytesPerChecksum, boolean includeMVCCReadpoint)
    throws IOException {
  this.dataBlockEncoder =
      dataBlockEncoder != null ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
  writer =
      HFile.getWriterFactory(conf, cacheConf).withPath(fs, path).withBlockSize(blocksize)
          .withCompression(compress).withDataBlockEncoder(dataBlockEncoder)
          .withComparator(comparator.getRawComparator()).withChecksumType(checksumType)
          .withBytesPerChecksum(bytesPerChecksum).includeMVCCReadpoint(includeMVCCReadpoint)
          .create();

  this.kvComparator = comparator;

  generalBloomFilterWriter =
      BloomFilterFactory.createGeneralBloomAtWrite(conf, cacheConf, bloomType,
        (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);

  if (generalBloomFilterWriter != null) {
    this.bloomType = bloomType;
    LOG.info("Bloom filter type for " + path + ": " + this.bloomType + ", "
        + generalBloomFilterWriter.getClass().getSimpleName());
  } else {
    // Not using Bloom filters.
    this.bloomType = BloomType.NONE;
  }

  // initialize delete family Bloom filter when there is NO RowCol Bloom
  // filter
  if (this.bloomType != BloomType.ROWCOL) {
    this.deleteFamilyBloomFilterWriter =
        BloomFilterFactory.createDeleteBloomAtWrite(conf, cacheConf,
          (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);
  } else {
    deleteFamilyBloomFilterWriter = null;
  }
  if (deleteFamilyBloomFilterWriter != null) {
    WinterOptimizer.ReplaceRawCode("Delete Family Bloom filter type for " + path + ": "
        + deleteFamilyBloomFilterWriter.getClass().getSimpleName());
    // LOG.info("Delete Family Bloom filter type for " + path + ": "
    // + deleteFamilyBloomFilterWriter.getClass().getSimpleName());
  }
  this.checksumType = checksumType;
  this.bytesPerChecksum = bytesPerChecksum;
}
 

import org.apache.hadoop.hbase.util.BloomFilterFactory; //导入方法依赖的package包/类
/**
 * Creates an HFile.Writer that also write helpful meta data.
 * @param fs file system to write to
 * @param path file name to create
 * @param conf user configuration
 * @param comparator key comparator
 * @param bloomType bloom filter setting
 * @param maxKeys the expected maximum number of keys to be added. Was used
 *        for Bloom filter size in {@link HFile} format version 1.
 * @param favoredNodes
 * @param fileContext - The HFile context
 * @throws IOException problem writing to FS
 */
private Writer(FileSystem fs, Path path,
               final Configuration conf,
               CacheConfig cacheConf,
               final KVComparator comparator, BloomType bloomType, long maxKeys,
               InetSocketAddress[] favoredNodes, HFileContext fileContext)
        throws IOException {
    writer = HFile.getWriterFactory(conf, cacheConf)
            .withPath(fs, path)
            .withComparator(comparator)
            .withFavoredNodes(favoredNodes)
            .withFileContext(fileContext)
            .create();

    this.kvComparator = comparator;

    generalBloomFilterWriter = BloomFilterFactory.createGeneralBloomAtWrite(
            conf, cacheConf, bloomType,
            (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);

    if (generalBloomFilterWriter != null) {
        this.bloomType = bloomType;
        if (LOG.isTraceEnabled()) LOG.trace("Bloom filter type for " + path + ": " +
                this.bloomType + ", " + generalBloomFilterWriter.getClass().getSimpleName());
    } else {
        // Not using Bloom filters.
        this.bloomType = BloomType.NONE;
    }

    // initialize delete family Bloom filter when there is NO RowCol Bloom
    // filter
    if (this.bloomType != BloomType.ROWCOL) {
        this.deleteFamilyBloomFilterWriter = BloomFilterFactory
                .createDeleteBloomAtWrite(conf, cacheConf,
                        (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);
    } else {
        deleteFamilyBloomFilterWriter = null;
    }
    if (deleteFamilyBloomFilterWriter != null) {
        if (LOG.isTraceEnabled()) LOG.trace("Delete Family Bloom filter type for " + path + ": "
                + deleteFamilyBloomFilterWriter.getClass().getSimpleName());
    }
}
 

import org.apache.hadoop.hbase.util.BloomFilterFactory; //导入方法依赖的package包/类
/**
 * Creates an HFile.Writer that also write helpful meta data.
 * @param fs file system to write to
 * @param path file name to create
 * @param conf user configuration
 * @param comparator key comparator
 * @param bloomType bloom filter setting
 * @param maxKeys the expected maximum number of keys to be added. Was used
 *        for Bloom filter size in {@link HFile} format version 1.
 * @param favoredNodes
 * @param fileContext - The HFile context
 * @throws IOException problem writing to FS
 */
private Writer(FileSystem fs, Path path,
    final Configuration conf,
    CacheConfig cacheConf,
    final KVComparator comparator, BloomType bloomType, long maxKeys,
    InetSocketAddress[] favoredNodes, HFileContext fileContext) 
        throws IOException {
  writer = HFile.getWriterFactory(conf, cacheConf)
      .withPath(fs, path)
      .withComparator(comparator)
      .withFavoredNodes(favoredNodes)
      .withFileContext(fileContext)
      .create();

  this.kvComparator = comparator;

  generalBloomFilterWriter = BloomFilterFactory.createGeneralBloomAtWrite(
      conf, cacheConf, bloomType,
      (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);

  if (generalBloomFilterWriter != null) {
    this.bloomType = bloomType;
    if (LOG.isTraceEnabled()) LOG.trace("Bloom filter type for " + path + ": " +
      this.bloomType + ", " + generalBloomFilterWriter.getClass().getSimpleName());
  } else {
    // Not using Bloom filters.
    this.bloomType = BloomType.NONE;
  }

  // initialize delete family Bloom filter when there is NO RowCol Bloom
  // filter
  if (this.bloomType != BloomType.ROWCOL) {
    this.deleteFamilyBloomFilterWriter = BloomFilterFactory
        .createDeleteBloomAtWrite(conf, cacheConf,
            (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);
  } else {
    deleteFamilyBloomFilterWriter = null;
  }
  if (deleteFamilyBloomFilterWriter != null) {
    if (LOG.isTraceEnabled()) LOG.trace("Delete Family Bloom filter type for " + path + ": "
        + deleteFamilyBloomFilterWriter.getClass().getSimpleName());
  }
}
 

import org.apache.hadoop.hbase.util.BloomFilterFactory; //导入方法依赖的package包/类
/**
 * Creates an HFile.Writer that also write helpful meta data.
 * @param fs file system to write to
 * @param path file name to create
 * @param blocksize HDFS block size
 * @param compress HDFS block compression
 * @param conf user configuration
 * @param comparator key comparator
 * @param bloomType bloom filter setting
 * @param maxKeys the expected maximum number of keys to be added. Was used
 *        for Bloom filter size in {@link HFile} format version 1.
 * @param checksumType the checksum type
 * @param bytesPerChecksum the number of bytes per checksum value
 * @param includeMVCCReadpoint whether to write the mvcc readpoint to the file for each KV
 * @throws IOException problem writing to FS
 */
private Writer(FileSystem fs, Path path, int blocksize,
    Compression.Algorithm compress,
    HFileDataBlockEncoder dataBlockEncoder, final Configuration conf,
    CacheConfig cacheConf,
    final KVComparator comparator, BloomType bloomType, long maxKeys,
    final ChecksumType checksumType, final int bytesPerChecksum, boolean includeMVCCReadpoint)
    throws IOException {
  this.dataBlockEncoder = dataBlockEncoder != null ?
      dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
  writer = HFile.getWriterFactory(conf, cacheConf)
      .withPath(fs, path)
      .withBlockSize(blocksize)
      .withCompression(compress)
      .withDataBlockEncoder(dataBlockEncoder)
      .withComparator(comparator.getRawComparator())
      .withChecksumType(checksumType)
      .withBytesPerChecksum(bytesPerChecksum)
      .includeMVCCReadpoint(includeMVCCReadpoint)
      .create();

  this.kvComparator = comparator;

  generalBloomFilterWriter = BloomFilterFactory.createGeneralBloomAtWrite(
      conf, cacheConf, bloomType,
      (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);

  if (generalBloomFilterWriter != null) {
    this.bloomType = bloomType;
    LOG.info("Bloom filter type for " + path + ": " + this.bloomType + ", "
        + generalBloomFilterWriter.getClass().getSimpleName());
  } else {
    // Not using Bloom filters.
    this.bloomType = BloomType.NONE;
  }

  // initialize delete family Bloom filter when there is NO RowCol Bloom
  // filter
  if (this.bloomType != BloomType.ROWCOL) {
    this.deleteFamilyBloomFilterWriter = BloomFilterFactory
        .createDeleteBloomAtWrite(conf, cacheConf,
            (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);
  } else {
    deleteFamilyBloomFilterWriter = null;
  }
  if (deleteFamilyBloomFilterWriter != null) {
    LOG.info("Delete Family Bloom filter type for " + path + ": "
        + deleteFamilyBloomFilterWriter.getClass().getSimpleName());
  }
  this.checksumType = checksumType;
  this.bytesPerChecksum = bytesPerChecksum;
}
 

import org.apache.hadoop.hbase.util.BloomFilterFactory; //导入方法依赖的package包/类
/**
   * Creates an HFile.Writer that also write helpful meta data.
   * @param fs file system to write to
   * @param path file name to create
   * @param conf user configuration
   * @param comparator key comparator
   * @param bloomType bloom filter setting
   * @param maxKeys the expected maximum number of keys to be added. Was used
   *        for Bloom filter size in {@link HFile} format version 1.
   * @param favoredNodes
   * @param fileContext - The HFile context
   * @param shouldDropCacheBehind Drop pages written to page cache after writing the store file.
   * @throws IOException problem writing to FS
   */
  private StoreFileWriter(FileSystem fs, Path path,
      final Configuration conf,
      CacheConfig cacheConf,
      final CellComparator comparator, BloomType bloomType, long maxKeys,
      InetSocketAddress[] favoredNodes, HFileContext fileContext,
      boolean shouldDropCacheBehind)
          throws IOException {
  this.timeRangeTracker = TimeRangeTracker.create(TimeRangeTracker.Type.NON_SYNC);
  // TODO : Change all writers to be specifically created for compaction context
  writer = HFile.getWriterFactory(conf, cacheConf)
      .withPath(fs, path)
      .withComparator(comparator)
      .withFavoredNodes(favoredNodes)
      .withFileContext(fileContext)
      .withShouldDropCacheBehind(shouldDropCacheBehind)
      .create();

  generalBloomFilterWriter = BloomFilterFactory.createGeneralBloomAtWrite(
      conf, cacheConf, bloomType,
      (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);

  if (generalBloomFilterWriter != null) {
    this.bloomType = bloomType;
    if (LOG.isTraceEnabled()) {
      LOG.trace("Bloom filter type for " + path + ": " + this.bloomType + ", " +
          generalBloomFilterWriter.getClass().getSimpleName());
    }
    // init bloom context
    switch (bloomType) {
    case ROW:
      bloomContext = new RowBloomContext(generalBloomFilterWriter, comparator);
      break;
    case ROWCOL:
      bloomContext = new RowColBloomContext(generalBloomFilterWriter, comparator);
      break;
    default:
      throw new IOException(
          "Invalid Bloom filter type: " + bloomType + " (ROW or ROWCOL expected)");
    }
  } else {
    // Not using Bloom filters.
    this.bloomType = BloomType.NONE;
  }

  // initialize delete family Bloom filter when there is NO RowCol Bloom
  // filter
  if (this.bloomType != BloomType.ROWCOL) {
    this.deleteFamilyBloomFilterWriter = BloomFilterFactory
        .createDeleteBloomAtWrite(conf, cacheConf,
            (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);
    deleteFamilyBloomContext = new RowBloomContext(deleteFamilyBloomFilterWriter, comparator);
  } else {
    deleteFamilyBloomFilterWriter = null;
  }
  if (deleteFamilyBloomFilterWriter != null && LOG.isTraceEnabled()) {
    LOG.trace("Delete Family Bloom filter type for " + path + ": " +
        deleteFamilyBloomFilterWriter.getClass().getSimpleName());
  }
}
 

import org.apache.hadoop.hbase.util.BloomFilterFactory; //导入方法依赖的package包/类
/**
 * Creates an HFile.Writer that also write helpful meta data.
 * @param fs file system to write to
 * @param path file name to create
 * @param conf user configuration
 * @param comparator key comparator
 * @param bloomType bloom filter setting
 * @param maxKeys the expected maximum number of keys to be added. Was used
 *        for Bloom filter size in {@link HFile} format version 1.
 * @param favoredNodes
 * @param fileContext - The HFile context
 * @throws IOException problem writing to FS
 */
private Writer(FileSystem fs, Path path,
    final Configuration conf,
    CacheConfig cacheConf,
    final KVComparator comparator, BloomType bloomType, long maxKeys,
    InetSocketAddress[] favoredNodes, HFileContext fileContext)
        throws IOException {
  //LOG.info("Shen Li: in Writer constructor");
  writer = HFile.getWriterFactory(conf, cacheConf)
      .withPath(fs, path)
      .withComparator(comparator)
      .withFavoredNodes(favoredNodes)
      .withFileContext(fileContext)
      .create();

  this.kvComparator = comparator;

  generalBloomFilterWriter = BloomFilterFactory.createGeneralBloomAtWrite(
      conf, cacheConf, bloomType,
      (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);

  if (generalBloomFilterWriter != null) {
    this.bloomType = bloomType;
    if (LOG.isTraceEnabled()) LOG.trace("Bloom filter type for " + path + ": " +
      this.bloomType + ", " + generalBloomFilterWriter.getClass().getSimpleName());
  } else {
    // Not using Bloom filters.
    this.bloomType = BloomType.NONE;
  }

  // initialize delete family Bloom filter when there is NO RowCol Bloom
  // filter
  if (this.bloomType != BloomType.ROWCOL) {
    this.deleteFamilyBloomFilterWriter = BloomFilterFactory
        .createDeleteBloomAtWrite(conf, cacheConf,
            (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);
  } else {
    deleteFamilyBloomFilterWriter = null;
  }
  if (deleteFamilyBloomFilterWriter != null) {
    if (LOG.isTraceEnabled()) LOG.trace("Delete Family Bloom filter type for " + path + ": "
        + deleteFamilyBloomFilterWriter.getClass().getSimpleName());
  }
}
 

import org.apache.hadoop.hbase.util.BloomFilterFactory; //导入方法依赖的package包/类
/**
 * Creates an HFile.Writer that also write helpful meta data.
 * @param fs file system to write to
 * @param path file name to create
 * @param blocksize HDFS block size
 * @param compress HDFS block compression
 * @param conf user configuration
 * @param comparator key comparator
 * @param bloomType bloom filter setting
 * @param maxKeys the expected maximum number of keys to be added. Was used
 *        for Bloom filter size in {@link HFile} format version 1.
 * @param checksumType the checksum type
 * @param bytesPerChecksum the number of bytes per checksum value
 * @param includeMVCCReadpoint whether to write the mvcc readpoint to the file for each KV
 * @param favoredNodes
 * @throws IOException problem writing to FS
 */
private Writer(FileSystem fs, Path path, int blocksize,
    Compression.Algorithm compress,
    HFileDataBlockEncoder dataBlockEncoder, final Configuration conf,
    CacheConfig cacheConf,
    final KVComparator comparator, BloomType bloomType, long maxKeys,
    final ChecksumType checksumType, final int bytesPerChecksum,
    final boolean includeMVCCReadpoint, InetSocketAddress[] favoredNodes)
        throws IOException {
  this.dataBlockEncoder = dataBlockEncoder != null ?
      dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
  writer = HFile.getWriterFactory(conf, cacheConf)
      .withPath(fs, path)
      .withBlockSize(blocksize)
      .withCompression(compress)
      .withDataBlockEncoder(this.dataBlockEncoder)
      .withComparator(comparator)
      .withChecksumType(checksumType)
      .withBytesPerChecksum(bytesPerChecksum)
      .withFavoredNodes(favoredNodes)
      .includeMVCCReadpoint(includeMVCCReadpoint)
      .create();

  this.kvComparator = comparator;

  generalBloomFilterWriter = BloomFilterFactory.createGeneralBloomAtWrite(
      conf, cacheConf, bloomType,
      (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);

  if (generalBloomFilterWriter != null) {
    this.bloomType = bloomType;
    if (LOG.isTraceEnabled()) LOG.trace("Bloom filter type for " + path + ": " +
      this.bloomType + ", " + generalBloomFilterWriter.getClass().getSimpleName());
  } else {
    // Not using Bloom filters.
    this.bloomType = BloomType.NONE;
  }

  // initialize delete family Bloom filter when there is NO RowCol Bloom
  // filter
  if (this.bloomType != BloomType.ROWCOL) {
    this.deleteFamilyBloomFilterWriter = BloomFilterFactory
        .createDeleteBloomAtWrite(conf, cacheConf,
            (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);
  } else {
    deleteFamilyBloomFilterWriter = null;
  }
  if (deleteFamilyBloomFilterWriter != null) {
    if (LOG.isTraceEnabled()) LOG.trace("Delete Family Bloom filter type for " + path + ": "
        + deleteFamilyBloomFilterWriter.getClass().getSimpleName());
  }
  this.checksumType = checksumType;
  this.bytesPerChecksum = bytesPerChecksum;
}
 

import org.apache.hadoop.hbase.util.BloomFilterFactory; //导入方法依赖的package包/类
/**
 * Creates an HFile.Writer that also write helpful meta data.
 * @param fs file system to write to
 * @param path file name to create
 * @param blocksize HDFS block size
 * @param compress HDFS block compression
 * @param conf user configuration
 * @param comparator key comparator
 * @param bloomType bloom filter setting
 * @param maxKeys the expected maximum number of keys to be added. Was used
 *        for Bloom filter size in {@link HFile} format version 1.
 * @param checksumType the checksum type
 * @param bytesPerChecksum the number of bytes per checksum value
 * @throws IOException problem writing to FS
 */
private Writer(FileSystem fs, Path path, int blocksize,
    Compression.Algorithm compress,
    HFileDataBlockEncoder dataBlockEncoder, final Configuration conf,
    CacheConfig cacheConf,
    final KVComparator comparator, BloomType bloomType, long maxKeys,
    final ChecksumType checksumType, final int bytesPerChecksum)
    throws IOException {
  this.dataBlockEncoder = dataBlockEncoder != null ?
      dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
  writer = HFile.getWriterFactory(conf, cacheConf)
      .withPath(fs, path)
      .withBlockSize(blocksize)
      .withCompression(compress)
      .withDataBlockEncoder(this.dataBlockEncoder)
      .withComparator(comparator.getRawComparator())
      .withChecksumType(checksumType)
      .withBytesPerChecksum(bytesPerChecksum)
      .create();

  this.kvComparator = comparator;

  generalBloomFilterWriter = BloomFilterFactory.createGeneralBloomAtWrite(
      conf, cacheConf, bloomType,
      (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);

  if (generalBloomFilterWriter != null) {
    this.bloomType = bloomType;
    LOG.info("Bloom filter type for " + path + ": " + this.bloomType + ", "
        + generalBloomFilterWriter.getClass().getSimpleName());
  } else {
    // Not using Bloom filters.
    this.bloomType = BloomType.NONE;
  }

  // initialize delete family Bloom filter when there is NO RowCol Bloom
  // filter
  if (this.bloomType != BloomType.ROWCOL) {
    this.deleteFamilyBloomFilterWriter = BloomFilterFactory
        .createDeleteBloomAtWrite(conf, cacheConf,
            (int) Math.min(maxKeys, Integer.MAX_VALUE), writer);
  } else {
    deleteFamilyBloomFilterWriter = null;
  }
  if (deleteFamilyBloomFilterWriter != null) {
    LOG.info("Delete Family Bloom filter type for " + path + ": "
        + deleteFamilyBloomFilterWriter.getClass().getSimpleName());
  }
  this.checksumType = checksumType;
  this.bytesPerChecksum = bytesPerChecksum;
}