Java HConstants.HFILEBLOCK_HEADER_SIZE屬性代碼示例

/**
 * @param dataBlockEncoder data block encoding algorithm to use
 */
public Writer(HFileDataBlockEncoder dataBlockEncoder, HFileContext fileContext) {
  this.dataBlockEncoder = dataBlockEncoder != null
      ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
  defaultBlockEncodingCtx = new HFileBlockDefaultEncodingContext(null,
      HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext);
  dataBlockEncodingCtx = this.dataBlockEncoder
      .newDataBlockEncodingContext(HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext);

  if (fileContext.getBytesPerChecksum() < HConstants.HFILEBLOCK_HEADER_SIZE) {
    throw new RuntimeException("Unsupported value of bytesPerChecksum. " +
        " Minimum is " + HConstants.HFILEBLOCK_HEADER_SIZE + " but the configured value is " +
        fileContext.getBytesPerChecksum());
  }

  baosInMemory = new ByteArrayOutputStream();

  prevOffsetByType = new long[BlockType.values().length];
  for (int i = 0; i < prevOffsetByType.length; ++i)
    prevOffsetByType[i] = -1;

  this.fileContext = fileContext;
}
 

/**
 * Maps a minor version to the size of the header.
 */
public static int headerSize(boolean usesHBaseChecksum) {
  if (usesHBaseChecksum) {
    return HConstants.HFILEBLOCK_HEADER_SIZE;
  }
  return HConstants.HFILEBLOCK_HEADER_SIZE_NO_CHECKSUM;
}
 

public String createTestBlockStr(Compression.Algorithm algo,
    int correctLength, boolean useTag) throws IOException {
  HFileBlock.Writer hbw = createTestV2Block(algo, includesMemstoreTS, useTag);
  byte[] testV2Block = hbw.getHeaderAndDataForTest();
  int osOffset = HConstants.HFILEBLOCK_HEADER_SIZE + 9;
  if (testV2Block.length == correctLength) {
    // Force-set the "OS" field of the gzip header to 3 (Unix) to avoid
    // variations across operating systems.
    // See http://www.gzip.org/zlib/rfc-gzip.html for gzip format.
    // We only make this change when the compressed block length matches.
    // Otherwise, there are obviously other inconsistencies.
    testV2Block[osOffset] = 3;
  }
  return Bytes.toStringBinary(testV2Block);
}
 

protected void testBlockHeapSizeInternals() {
  if (ClassSize.is32BitJVM()) {
    assertTrue(HFileBlock.BYTE_BUFFER_HEAP_SIZE == 64);
  } else {
    assertTrue(HFileBlock.BYTE_BUFFER_HEAP_SIZE == 80);
  }

  for (int size : new int[] { 100, 256, 12345 }) {
    byte[] byteArr = new byte[HConstants.HFILEBLOCK_HEADER_SIZE + size];
    ByteBuffer buf = ByteBuffer.wrap(byteArr, 0, size);
    HFileContext meta = new HFileContextBuilder()
                        .withIncludesMvcc(includesMemstoreTS)
                        .withIncludesTags(includesTag)
                        .withHBaseCheckSum(false)
                        .withCompression(Algorithm.NONE)
                        .withBytesPerCheckSum(HFile.DEFAULT_BYTES_PER_CHECKSUM)
                        .withChecksumType(ChecksumType.NULL).build();
    HFileBlock block = new HFileBlock(BlockType.DATA, size, size, -1, buf,
        HFileBlock.FILL_HEADER, -1, 
        0, meta);
    long byteBufferExpectedSize =
        ClassSize.align(ClassSize.estimateBase(buf.getClass(), true)
            + HConstants.HFILEBLOCK_HEADER_SIZE + size);
    long hfileMetaSize =  ClassSize.align(ClassSize.estimateBase(HFileContext.class, true));
    long hfileBlockExpectedSize =
        ClassSize.align(ClassSize.estimateBase(HFileBlock.class, true));
    long expected = hfileBlockExpectedSize + byteBufferExpectedSize + hfileMetaSize;
    assertEquals("Block data size: " + size + ", byte buffer expected " +
        "size: " + byteBufferExpectedSize + ", HFileBlock class expected " +
        "size: " + hfileBlockExpectedSize + ";", expected,
        block.heapSize());
  }
}
 

/**
 * The uncompressed size of the block data. Does not include header size.
 */
int getUncompressedSizeWithoutHeader() {
  expectState(State.BLOCK_READY);
  return uncompressedBytesWithHeader.length - HConstants.HFILEBLOCK_HEADER_SIZE;
}
 

private static HFileBlockPair[] generateHFileBlocks(int blockSize,
    int numBlocks) {
  HFileBlockPair[] returnedBlocks = new HFileBlockPair[numBlocks];
  Random rand = new Random();
  HashSet<String> usedStrings = new HashSet<String>();
  for (int i = 0; i < numBlocks; i++) {

    // The buffer serialized size needs to match the size of BlockSize. So we
    // declare our data size to be smaller than it by the serialization space
    // required.

    ByteBuffer cachedBuffer = ByteBuffer.allocate(blockSize
        - HFileBlock.EXTRA_SERIALIZATION_SPACE);
    rand.nextBytes(cachedBuffer.array());
    cachedBuffer.rewind();
    int onDiskSizeWithoutHeader = blockSize
        - HFileBlock.EXTRA_SERIALIZATION_SPACE;
    int uncompressedSizeWithoutHeader = blockSize
        - HFileBlock.EXTRA_SERIALIZATION_SPACE;
    long prevBlockOffset = rand.nextLong();
    BlockType.DATA.write(cachedBuffer);
    cachedBuffer.putInt(onDiskSizeWithoutHeader);
    cachedBuffer.putInt(uncompressedSizeWithoutHeader);
    cachedBuffer.putLong(prevBlockOffset);
    cachedBuffer.rewind();
    HFileContext meta = new HFileContextBuilder()
                        .withHBaseCheckSum(false)
                        .withIncludesMvcc(includesMemstoreTS)
                        .withIncludesTags(false)
                        .withCompression(Compression.Algorithm.NONE)
                        .withBytesPerCheckSum(0)
                        .withChecksumType(ChecksumType.NULL)
                        .build();
    HFileBlock generated = new HFileBlock(BlockType.DATA,
        onDiskSizeWithoutHeader, uncompressedSizeWithoutHeader,
        prevBlockOffset, cachedBuffer, HFileBlock.DONT_FILL_HEADER,
        blockSize,
        onDiskSizeWithoutHeader + HConstants.HFILEBLOCK_HEADER_SIZE, meta);

    String strKey;
    /* No conflicting keys */
    for (strKey = new Long(rand.nextLong()).toString(); !usedStrings
        .add(strKey); strKey = new Long(rand.nextLong()).toString())
      ;

    returnedBlocks[i] = new HFileBlockPair();
    returnedBlocks[i].blockName = new BlockCacheKey(strKey, 0);
    returnedBlocks[i].block = generated;
  }
  return returnedBlocks;
}
 

/**
 * Returns the on-disk size of the data portion of the block. This is the
 * compressed size if compression is enabled. Can only be called in the
 * "block ready" state. Header is not compressed, and its size is not
 * included in the return value.
 *
 * @return the on-disk size of the block, not including the header.
 */
int getOnDiskSizeWithoutHeader() {
  expectState(State.BLOCK_READY);
  return onDiskBytesWithHeader.length
      + onDiskChecksum.length
      - HConstants.HFILEBLOCK_HEADER_SIZE;
}