Java DataBlockEncoding.NONE属性代码示例

public static HFileDataBlockEncoder createFromFileInfo(
    FileInfo fileInfo) throws IOException {
  DataBlockEncoding encoding = DataBlockEncoding.NONE;
  byte[] dataBlockEncodingType = fileInfo.get(DATA_BLOCK_ENCODING);
  if (dataBlockEncodingType != null) {
    String dataBlockEncodingStr = Bytes.toString(dataBlockEncodingType);
    try {
      encoding = DataBlockEncoding.valueOf(dataBlockEncodingStr);
    } catch (IllegalArgumentException ex) {
      throw new IOException("Invalid data block encoding type in file info: "
        + dataBlockEncodingStr, ex);
    }
  }

  if (encoding == DataBlockEncoding.NONE) {
    return NoOpDataBlockEncoder.INSTANCE;
  }
  return new HFileDataBlockEncoderImpl(encoding);
}
 

public AbstractHFileWriter(CacheConfig cacheConf,
    FSDataOutputStream outputStream, Path path, 
    KVComparator comparator, HFileContext fileContext) {
  this.outputStream = outputStream;
  this.path = path;
  this.name = path != null ? path.getName() : outputStream.toString();
  this.hFileContext = fileContext;
  DataBlockEncoding encoding = hFileContext.getDataBlockEncoding();
  if (encoding != DataBlockEncoding.NONE) {
    this.blockEncoder = new HFileDataBlockEncoderImpl(encoding);
  } else {
    this.blockEncoder = NoOpDataBlockEncoder.INSTANCE;
  }
  this.comparator = comparator != null ? comparator
      : KeyValue.COMPARATOR;

  closeOutputStream = path != null;
  this.cacheConf = cacheConf;
}
 

public void majorCompactionWithDataBlockEncoding(boolean inCacheOnly)
    throws Exception {
  Map<Store, HFileDataBlockEncoder> replaceBlockCache =
      new HashMap<Store, HFileDataBlockEncoder>();
  for (Store store : r.getStores()) {
    HFileDataBlockEncoder blockEncoder = store.getDataBlockEncoder();
    replaceBlockCache.put(store, blockEncoder);
    final DataBlockEncoding inCache = DataBlockEncoding.PREFIX;
    final DataBlockEncoding onDisk = inCacheOnly ? DataBlockEncoding.NONE :
        inCache;
    ((HStore)store).setDataBlockEncoderInTest(new HFileDataBlockEncoderImpl(onDisk));
  }

  majorCompaction();

  // restore settings
  for (Entry<Store, HFileDataBlockEncoder> entry : replaceBlockCache.entrySet()) {
    ((HStore)entry.getKey()).setDataBlockEncoderInTest(entry.getValue());
  }
}
 

private void testEncodingWithCacheInternals(boolean useTag) throws IOException {
  List<KeyValue> kvs = generator.generateTestKeyValues(60, useTag);
  HFileBlock block = getSampleHFileBlock(kvs, useTag);
  HFileBlock cacheBlock = createBlockOnDisk(kvs, block, useTag);

  LruBlockCache blockCache =
      new LruBlockCache(8 * 1024 * 1024, 32 * 1024);
  BlockCacheKey cacheKey = new BlockCacheKey("test", 0);
  blockCache.cacheBlock(cacheKey, cacheBlock);

  HeapSize heapSize = blockCache.getBlock(cacheKey, false, false, true);
  assertTrue(heapSize instanceof HFileBlock);

  HFileBlock returnedBlock = (HFileBlock) heapSize;;

  if (blockEncoder.getDataBlockEncoding() ==
      DataBlockEncoding.NONE) {
    assertEquals(block.getBufferWithHeader(),
        returnedBlock.getBufferWithHeader());
  } else {
    if (BlockType.ENCODED_DATA != returnedBlock.getBlockType()) {
      System.out.println(blockEncoder);
    }
    assertEquals(BlockType.ENCODED_DATA, returnedBlock.getBlockType());
  }
}
 

/**
 * @return All possible data block encoding configurations
 */
@Parameters
public static Collection<Object[]> getAllConfigurations() {
  List<Object[]> configurations =
      new ArrayList<Object[]>();

  for (DataBlockEncoding diskAlgo : DataBlockEncoding.values()) {
    for (boolean includesMemstoreTS : new boolean[] { false, true }) {
      HFileDataBlockEncoder dbe = (diskAlgo == DataBlockEncoding.NONE) ? 
          NoOpDataBlockEncoder.INSTANCE : new HFileDataBlockEncoderImpl(diskAlgo);
      configurations.add(new Object[] { dbe, new Boolean(includesMemstoreTS) });
    }
  }

  return configurations;
}
 

/**
 * Test writing to disk.
 */
@Test
public void testEncodingWritePath() {
  // usually we have just block without headers, but don't complicate that
  HFileBlock block = getSampleHFileBlock();
  Pair<ByteBuffer, BlockType> result =
      blockEncoder.beforeWriteToDisk(block.getBufferWithoutHeader(),
          includesMemstoreTS, HFileBlock.DUMMY_HEADER_WITH_CHECKSUM);

  int size = result.getFirst().limit() - HFileBlock.HEADER_SIZE_WITH_CHECKSUMS;
  HFileBlock blockOnDisk = new HFileBlock(result.getSecond(),
      size, size, -1, result.getFirst(), HFileBlock.FILL_HEADER, 0,
      includesMemstoreTS, block.getMinorVersion(),
      block.getBytesPerChecksum(), block.getChecksumType(),
      block.getOnDiskDataSizeWithHeader());

  if (blockEncoder.getEncodingOnDisk() !=
      DataBlockEncoding.NONE) {
    assertEquals(BlockType.ENCODED_DATA, blockOnDisk.getBlockType());
    assertEquals(blockEncoder.getEncodingOnDisk().getId(),
        blockOnDisk.getDataBlockEncodingId());
  } else {
    assertEquals(BlockType.DATA, blockOnDisk.getBlockType());
  }
}
 

private void testEncodingWithCacheInternals(boolean useTag) throws IOException {
  HFileBlock block = getSampleHFileBlock(useTag);
  HFileBlock cacheBlock = createBlockOnDisk(block, useTag);

  LruBlockCache blockCache =
      new LruBlockCache(8 * 1024 * 1024, 32 * 1024);
  BlockCacheKey cacheKey = new BlockCacheKey("test", 0);
  blockCache.cacheBlock(cacheKey, cacheBlock);

  HeapSize heapSize = blockCache.getBlock(cacheKey, false, false, true);
  assertTrue(heapSize instanceof HFileBlock);

  HFileBlock returnedBlock = (HFileBlock) heapSize;;

  if (blockEncoder.getDataBlockEncoding() ==
      DataBlockEncoding.NONE) {
    assertEquals(block.getBufferWithHeader(),
        returnedBlock.getBufferWithHeader());
  } else {
    if (BlockType.ENCODED_DATA != returnedBlock.getBlockType()) {
      System.out.println(blockEncoder);
    }
    assertEquals(BlockType.ENCODED_DATA, returnedBlock.getBlockType());
  }
}
 

protected void parseColumnFamilyOptions(CommandLine cmd) {
  String dataBlockEncodingStr = cmd.getOptionValue(OPT_DATA_BLOCK_ENCODING);
  dataBlockEncodingAlgo = dataBlockEncodingStr == null ? null :
      DataBlockEncoding.valueOf(dataBlockEncodingStr);
  if (dataBlockEncodingAlgo == DataBlockEncoding.NONE && encodeInCacheOnly) {
    throw new IllegalArgumentException("-" + OPT_ENCODE_IN_CACHE_ONLY + " " +
        "does not make sense when data block encoding is not used");
  }

  String compressStr = cmd.getOptionValue(OPT_COMPRESSION);
  compressAlgo = compressStr == null ? Compression.Algorithm.NONE :
      Compression.Algorithm.valueOf(compressStr);

  String bloomStr = cmd.getOptionValue(OPT_BLOOM);
  bloomType = bloomStr == null ? null :
      StoreFile.BloomType.valueOf(bloomStr);

  inMemoryCF = cmd.hasOption(OPT_INMEMORY);
}
 

/**
 * Serialize column family to data block encoding map to configuration.
 * Invoked while configuring the MR job for incremental load.
 *
 * @param table to read the properties from
 * @param conf to persist serialized values into
 * @throws IOException
 *           on failure to read column family descriptors
 */
@VisibleForTesting
static void configureDataBlockEncoding(HTableDescriptor tableDescriptor,
    Configuration conf) throws UnsupportedEncodingException {
  if (tableDescriptor == null) {
    // could happen with mock table instance
    return;
  }
  StringBuilder dataBlockEncodingConfigValue = new StringBuilder();
  Collection<HColumnDescriptor> families = tableDescriptor.getFamilies();
  int i = 0;
  for (HColumnDescriptor familyDescriptor : families) {
    if (i++ > 0) {
      dataBlockEncodingConfigValue.append('&');
    }
    dataBlockEncodingConfigValue.append(
        URLEncoder.encode(familyDescriptor.getNameAsString(), "UTF-8"));
    dataBlockEncodingConfigValue.append('=');
    DataBlockEncoding encoding = familyDescriptor.getDataBlockEncoding();
    if (encoding == null) {
      encoding = DataBlockEncoding.NONE;
    }
    dataBlockEncodingConfigValue.append(URLEncoder.encode(encoding.toString(),
        "UTF-8"));
  }
  conf.set(DATABLOCK_ENCODING_FAMILIES_CONF_KEY,
      dataBlockEncodingConfigValue.toString());
}
 

protected void finishFileInfo() throws IOException {
  super.finishFileInfo();
  if (hFileContext.getDataBlockEncoding() == DataBlockEncoding.PREFIX_TREE) {
    // In case of Prefix Tree encoding, we always write tags information into HFiles even if all
    // KVs are having no tags.
    fileInfo.append(FileInfo.MAX_TAGS_LEN, Bytes.toBytes(this.maxTagsLength), false);
  } else if (hFileContext.isIncludesTags()) {
    // When tags are not being written in this file, MAX_TAGS_LEN is excluded
    // from the FileInfo
    fileInfo.append(FileInfo.MAX_TAGS_LEN, Bytes.toBytes(this.maxTagsLength), false);
    boolean tagsCompressed = (hFileContext.getDataBlockEncoding() != DataBlockEncoding.NONE)
      && hFileContext.isCompressTags();
    fileInfo.append(FileInfo.TAGS_COMPRESSED, Bytes.toBytes(tagsCompressed), false);
  }
}
 

@Override
public DataBlockEncoding getEffectiveEncodingInCache(boolean isCompaction) {
  if (!useEncodedScanner(isCompaction)) {
    return DataBlockEncoding.NONE;
  }
  return encoding;
}
 

@Override
public void startBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out)
    throws IOException {
  if (this.encoding != null && this.encoding != DataBlockEncoding.NONE) {
    this.encoding.getEncoder().startBlockEncoding(encodingCtx, out);
  }
}
 

public TestMultiColumnScanner(Compression.Algorithm comprAlgo,
    BloomType bloomType, boolean useDataBlockEncoding) {
  this.comprAlgo = comprAlgo;
  this.bloomType = bloomType;
  this.dataBlockEncoding = useDataBlockEncoding ? DataBlockEncoding.PREFIX :
      DataBlockEncoding.NONE;
}
 

private void testEncodingInternals(boolean useTag) throws IOException {
  // usually we have just block without headers, but don't complicate that
  List<KeyValue> kvs = generator.generateTestKeyValues(60, useTag);
  HFileBlock block = getSampleHFileBlock(kvs, useTag);
  HFileBlock blockOnDisk = createBlockOnDisk(kvs, block, useTag);

  if (blockEncoder.getDataBlockEncoding() !=
      DataBlockEncoding.NONE) {
    assertEquals(BlockType.ENCODED_DATA, blockOnDisk.getBlockType());
    assertEquals(blockEncoder.getDataBlockEncoding().getId(),
        blockOnDisk.getDataBlockEncodingId());
  } else {
    assertEquals(BlockType.DATA, blockOnDisk.getBlockType());
  }
}
 

@Override
public void endBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out,
    byte[] uncompressedBytesWithHeader) throws IOException {
  PrefixTreeEncodingState state = (PrefixTreeEncodingState) encodingCtx.getEncodingState();
  PrefixTreeEncoder builder = state.builder;
  builder.flush();
  EncoderFactory.checkIn(builder);
  // do i need to check this, or will it always be DataBlockEncoding.PREFIX_TREE?
  if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
    encodingCtx.postEncoding(BlockType.ENCODED_DATA);
  } else {
    encodingCtx.postEncoding(BlockType.DATA);
  }
}