Java HFileBlockEncodingContext类代码示例

import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; //导入依赖的package包/类
@Override
public int encode(Cell cell, HFileBlockEncodingContext encodingCtx, DataOutputStream out)
    throws IOException {
  int klength = KeyValueUtil.keyLength(cell);
  int vlength = cell.getValueLength();

  out.writeInt(klength);
  out.writeInt(vlength);
  CellUtil.writeFlatKey(cell, out);
  out.write(cell.getValueArray(), cell.getValueOffset(), vlength);
  int encodedKvSize = klength + vlength + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE;
  // Write the additional tag into the stream
  if (encodingCtx.getHFileContext().isIncludesTags()) {
    int tagsLength = cell.getTagsLength();
    out.writeShort(tagsLength);
    if (tagsLength > 0) {
      out.write(cell.getTagsArray(), cell.getTagsOffset(), tagsLength);
    }
    encodedKvSize += tagsLength + KeyValue.TAGS_LENGTH_SIZE;
  }
  if (encodingCtx.getHFileContext().isIncludesMvcc()) {
    WritableUtils.writeVLong(out, cell.getSequenceId());
    encodedKvSize += WritableUtils.getVIntSize(cell.getSequenceId());
  }
  return encodedKvSize;
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; //导入依赖的package包/类
private HFileBlock createBlockOnDisk(List<KeyValue> kvs, HFileBlock block, boolean useTags)
    throws IOException {
  int size;
  HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext(
      blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER,
      block.getHFileContext());

  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  baos.write(block.getDummyHeaderForVersion());
  DataOutputStream dos = new DataOutputStream(baos);
  blockEncoder.startBlockEncoding(context, dos);
  for (KeyValue kv : kvs) {
    blockEncoder.encode(kv, context, dos);
  }
  BufferGrabbingByteArrayOutputStream stream = new BufferGrabbingByteArrayOutputStream();
  baos.writeTo(stream);
  blockEncoder.endBlockEncoding(context, dos, stream.getBuffer(), BlockType.DATA);
  byte[] encodedBytes = baos.toByteArray();
  size = encodedBytes.length - block.getDummyHeaderForVersion().length;
  return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes),
      HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), block.getHFileContext());
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; //导入依赖的package包/类
@Override
public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out)
    throws IOException {
  if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
    throw new IOException(this.getClass().getName() + " only accepts "
        + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context.");
  }

  HFileBlockDefaultEncodingContext encodingCtx = 
      (HFileBlockDefaultEncodingContext) blkEncodingCtx;
  encodingCtx.prepareEncoding(out);

  PrefixTreeEncoder builder = EncoderFactory.checkOut(out, encodingCtx.getHFileContext()
      .isIncludesMvcc());
  PrefixTreeEncodingState state = new PrefixTreeEncodingState();
  state.builder = builder;
  blkEncodingCtx.setEncodingState(state);
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; //导入依赖的package包/类
/**
 * Copied from BufferedDataBlockEncoder. Almost definitely can be improved, but i'm not familiar
 * enough with the concept of the HFileBlockEncodingContext.
 */
@Override
public void encodeKeyValues(ByteBuffer in,
    HFileBlockEncodingContext blkEncodingCtx) throws IOException {
  if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
    throw new IOException(this.getClass().getName() + " only accepts "
        + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context.");
  }

  HFileBlockDefaultEncodingContext encodingCtx
      = (HFileBlockDefaultEncodingContext) blkEncodingCtx;
  encodingCtx.prepareEncoding();
  DataOutputStream dataOut = encodingCtx.getOutputStreamForEncoder();
  internalEncodeKeyValues(dataOut, in, encodingCtx.getHFileContext().isIncludesMvcc(),
      encodingCtx.getHFileContext().isIncludesTags());

  //do i need to check this, or will it always be DataBlockEncoding.PREFIX_TREE?
  if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
    encodingCtx.postEncoding(BlockType.ENCODED_DATA);
  } else {
    encodingCtx.postEncoding(BlockType.DATA);
  }
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; //导入依赖的package包/类
@Override
public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx,
    DataOutputStream out) throws IOException {
  if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
    throw new IOException(this.getClass().getName() + " only accepts "
        + HFileBlockDefaultEncodingContext.class.getName() + " as the "
        + "encoding context.");
  }

  HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx;
  encodingCtx.prepareEncoding(out);

  NoneEncoder encoder = new NoneEncoder(out, encodingCtx);
  NoneEncodingState state = new NoneEncodingState();
  state.encoder = encoder;
  blkEncodingCtx.setEncodingState(state);
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; //导入依赖的package包/类
private HFileBlock createBlockOnDisk(List<KeyValue> kvs, HFileBlock block, boolean useTags)
    throws IOException {
  int size;
  HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext(
      blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER,
      block.getHFileContext());

  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  baos.write(block.getDummyHeaderForVersion());
  DataOutputStream dos = new DataOutputStream(baos);
  blockEncoder.startBlockEncoding(context, dos);
  for (KeyValue kv : kvs) {
    blockEncoder.encode(kv, context, dos);
  }
  blockEncoder.endBlockEncoding(context, dos, baos.getBuffer(), BlockType.DATA);
  byte[] encodedBytes = baos.toByteArray();
  size = encodedBytes.length - block.getDummyHeaderForVersion().length;
  return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes),
      HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), -1,
      block.getHFileContext());
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; //导入依赖的package包/类
@Override
public int encode(KeyValue kv, HFileBlockEncodingContext encodingCtx, DataOutputStream out)
    throws IOException {
  int klength = kv.getKeyLength();
  int vlength = kv.getValueLength();

  out.writeInt(klength);
  out.writeInt(vlength);
  out.write(kv.getBuffer(), kv.getKeyOffset(), klength);
  out.write(kv.getValueArray(), kv.getValueOffset(), vlength);
  int encodedKvSize = klength + vlength + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE;
  // Write the additional tag into the stream
  if (encodingCtx.getHFileContext().isIncludesTags()) {
    short tagsLength = kv.getTagsLength();
    out.writeShort(tagsLength);
    if (tagsLength > 0) {
      out.write(kv.getTagsArray(), kv.getTagsOffset(), tagsLength);
    }
    encodedKvSize += tagsLength + KeyValue.TAGS_LENGTH_SIZE;
  }
  if (encodingCtx.getHFileContext().isIncludesMvcc()) {
    WritableUtils.writeVLong(out, kv.getMvccVersion());
    encodedKvSize += WritableUtils.getVIntSize(kv.getMvccVersion());
  }
  return encodedKvSize;
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; //导入依赖的package包/类
/**
 * Copied from BufferedDataBlockEncoder. Almost definitely can be improved, but i'm not familiar
 * enough with the concept of the HFileBlockEncodingContext.
 */
@Override
public void encodeKeyValues(ByteBuffer in, boolean includesMvccVersion,
    HFileBlockEncodingContext blkEncodingCtx) throws IOException {
  if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
    throw new IOException(this.getClass().getName() + " only accepts "
        + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context.");
  }

  HFileBlockDefaultEncodingContext encodingCtx
      = (HFileBlockDefaultEncodingContext) blkEncodingCtx;
  encodingCtx.prepareEncoding();
  DataOutputStream dataOut = encodingCtx.getOutputStreamForEncoder();
  internalEncodeKeyValues(dataOut, in, includesMvccVersion);

  //do i need to check this, or will it always be DataBlockEncoding.PREFIX_TREE?
  if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
    encodingCtx.postEncoding(BlockType.ENCODED_DATA);
  } else {
    encodingCtx.postEncoding(BlockType.DATA);
  }
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; //导入依赖的package包/类
private HFileBlock encodeDataBlock(HFileBlock block,
    DataBlockEncoding algo, boolean includesMemstoreTS,
    HFileBlockEncodingContext encodingCtx) {
  encodeBufferToHFileBlockBuffer(
    block.getBufferWithoutHeader(), algo, includesMemstoreTS, encodingCtx);
  byte[] encodedUncompressedBytes =
    encodingCtx.getUncompressedBytesWithHeader();
  ByteBuffer bufferWrapper = ByteBuffer.wrap(encodedUncompressedBytes);
  int sizeWithoutHeader = bufferWrapper.limit() - encodingCtx.getHeaderSize();
  HFileBlock encodedBlock = new HFileBlock(BlockType.ENCODED_DATA,
      block.getOnDiskSizeWithoutHeader(),
      sizeWithoutHeader, block.getPrevBlockOffset(),
      bufferWrapper, HFileBlock.FILL_HEADER, block.getOffset(),
      includesMemstoreTS, block.getMinorVersion(),
      block.getBytesPerChecksum(), block.getChecksumType(),
      block.getOnDiskDataSizeWithHeader());
  return encodedBlock;
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; //导入依赖的package包/类
@Override
public HFileBlockEncodingContext newDataBlockEncodingContext(
    byte[] dummyHeader, HFileContext fileContext) {
  DataBlockEncoder encoder = encoding.getEncoder();
  if (encoder != null) {
    return encoder.newDataBlockEncodingContext(encoding, dummyHeader, fileContext);
  }
  return new HFileBlockDefaultEncodingContext(null, dummyHeader, fileContext);
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; //导入依赖的package包/类
@Override
public void startBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out)
    throws IOException {
  if (this.encoding != null && this.encoding != DataBlockEncoding.NONE) {
    this.encoding.getEncoder().startBlockEncoding(encodingCtx, out);
  }
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; //导入依赖的package包/类
@Override
public HFileBlockEncodingContext newDataBlockEncodingContext(
    DataBlockEncoding encoding, byte[] header, HFileContext meta) {
  if(DataBlockEncoding.PREFIX_TREE != encoding){
    //i'm not sure why encoding is in the interface.  Each encoder implementation should probably
    //know it's encoding type
    throw new IllegalArgumentException("only DataBlockEncoding.PREFIX_TREE supported");
  }
  return new HFileBlockDefaultEncodingContext(encoding, header, meta);
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; //导入依赖的package包/类
@Override
public int encode(Cell cell, HFileBlockEncodingContext encodingCtx, DataOutputStream out)
    throws IOException {
  PrefixTreeEncodingState state = (PrefixTreeEncodingState) encodingCtx.getEncodingState();
  PrefixTreeEncoder builder = state.builder;
  builder.write(cell);
  int size = KeyValueUtil.length(cell);
  if (encodingCtx.getHFileContext().isIncludesMvcc()) {
    size += WritableUtils.getVIntSize(cell.getSequenceId());
  }
  return size;
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; //导入依赖的package包/类
@Override
public void endBlockEncoding(HFileBlockEncodingContext encodingCtx, DataOutputStream out,
    byte[] uncompressedBytesWithHeader) throws IOException {
  PrefixTreeEncodingState state = (PrefixTreeEncodingState) encodingCtx.getEncodingState();
  PrefixTreeEncoder builder = state.builder;
  builder.flush();
  EncoderFactory.checkIn(builder);
  // do i need to check this, or will it always be DataBlockEncoding.PREFIX_TREE?
  if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
    encodingCtx.postEncoding(BlockType.ENCODED_DATA);
  } else {
    encodingCtx.postEncoding(BlockType.DATA);
  }
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockEncodingContext; //导入依赖的package包/类
/**
 * Precondition: a non-encoded buffer. Postcondition: on-disk encoding.
 *
 * The encoded results can be stored in {@link HFileBlockEncodingContext}.
 *
 * @throws IOException
 */
@Override
public void beforeWriteToDisk(ByteBuffer in,
    HFileBlockEncodingContext encodeCtx,
    BlockType blockType) throws IOException {
  if (encoding == DataBlockEncoding.NONE) {
    // there is no need to encode the block before writing it to disk
    ((HFileBlockDefaultEncodingContext) encodeCtx).compressAfterEncodingWithBlockType(
        in.array(), blockType);
    return;
  }
  encodeBufferToHFileBlockBuffer(in, encoding, encodeCtx);
}