Java HFileBlockDefaultEncodingContext类代码示例

import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; //导入依赖的package包/类
/**
 * @param dataBlockEncoder data block encoding algorithm to use
 */
public Writer(HFileDataBlockEncoder dataBlockEncoder, HFileContext fileContext) {
  this.dataBlockEncoder = dataBlockEncoder != null
      ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
  defaultBlockEncodingCtx = new HFileBlockDefaultEncodingContext(null,
      HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext);
  dataBlockEncodingCtx = this.dataBlockEncoder
      .newDataBlockEncodingContext(HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext);

  if (fileContext.getBytesPerChecksum() < HConstants.HFILEBLOCK_HEADER_SIZE) {
    throw new RuntimeException("Unsupported value of bytesPerChecksum. " +
        " Minimum is " + HConstants.HFILEBLOCK_HEADER_SIZE + " but the configured value is " +
        fileContext.getBytesPerChecksum());
  }

  baosInMemory = new ByteArrayOutputStream();

  prevOffsetByType = new long[BlockType.values().length];
  for (int i = 0; i < prevOffsetByType.length; ++i)
    prevOffsetByType[i] = -1;

  this.fileContext = fileContext;
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; //导入依赖的package包/类
private HFileBlock createBlockOnDisk(List<KeyValue> kvs, HFileBlock block, boolean useTags)
    throws IOException {
  int size;
  HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext(
      blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER,
      block.getHFileContext());

  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  baos.write(block.getDummyHeaderForVersion());
  DataOutputStream dos = new DataOutputStream(baos);
  blockEncoder.startBlockEncoding(context, dos);
  for (KeyValue kv : kvs) {
    blockEncoder.encode(kv, context, dos);
  }
  BufferGrabbingByteArrayOutputStream stream = new BufferGrabbingByteArrayOutputStream();
  baos.writeTo(stream);
  blockEncoder.endBlockEncoding(context, dos, stream.getBuffer(), BlockType.DATA);
  byte[] encodedBytes = baos.toByteArray();
  size = encodedBytes.length - block.getDummyHeaderForVersion().length;
  return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes),
      HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), block.getHFileContext());
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; //导入依赖的package包/类
@Override
public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx, DataOutputStream out)
    throws IOException {
  if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
    throw new IOException(this.getClass().getName() + " only accepts "
        + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context.");
  }

  HFileBlockDefaultEncodingContext encodingCtx = 
      (HFileBlockDefaultEncodingContext) blkEncodingCtx;
  encodingCtx.prepareEncoding(out);

  PrefixTreeEncoder builder = EncoderFactory.checkOut(out, encodingCtx.getHFileContext()
      .isIncludesMvcc());
  PrefixTreeEncodingState state = new PrefixTreeEncodingState();
  state.builder = builder;
  blkEncodingCtx.setEncodingState(state);
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; //导入依赖的package包/类
/**
 * @param dataBlockEncoder data block encoding algorithm to use
 */
public Writer(HFileDataBlockEncoder dataBlockEncoder, HFileContext fileContext) {
  this.dataBlockEncoder = dataBlockEncoder != null
      ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;
  defaultBlockEncodingCtx = new HFileBlockDefaultEncodingContext(null,
      HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext);
  dataBlockEncodingCtx = this.dataBlockEncoder
      .newDataBlockEncodingContext(HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext);

  if (fileContext.getBytesPerChecksum() < HConstants.HFILEBLOCK_HEADER_SIZE) {
    throw new RuntimeException("Unsupported value of bytesPerChecksum. " +
        " Minimum is " + HConstants.HFILEBLOCK_HEADER_SIZE + " but the configured value is " +
        fileContext.getBytesPerChecksum());
  }

  baosInMemory = new ByteArrayOutputStream();
  
  prevOffsetByType = new long[BlockType.values().length];
  for (int i = 0; i < prevOffsetByType.length; ++i)
    prevOffsetByType[i] = -1;

  this.fileContext = fileContext;
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; //导入依赖的package包/类
/**
 * @param compressionAlgorithm compression algorithm to use
 * @param dataBlockEncoderAlgo data block encoding algorithm to use
 */
public Writer(Compression.Algorithm compressionAlgorithm,
      HFileDataBlockEncoder dataBlockEncoder, boolean includesMemstoreTS, boolean includesTag) {
  compressAlgo = compressionAlgorithm == null ? NONE : compressionAlgorithm;
  this.dataBlockEncoder = dataBlockEncoder != null
      ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;

  meta = new HFileContextBuilder()
          .withHBaseCheckSum(false)
          .withIncludesMvcc(includesMemstoreTS)
          .withIncludesTags(includesTag)
          .withCompression(compressionAlgorithm)
          .build();
  defaultBlockEncodingCtx = new HFileBlockDefaultEncodingContext(null, DUMMY_HEADER, meta);
  dataBlockEncodingCtx =
      this.dataBlockEncoder.newDataBlockEncodingContext(
          DUMMY_HEADER, meta);
  baosInMemory = new ByteArrayOutputStream();

  prevOffsetByType = new long[BlockType.values().length];
  for (int i = 0; i < prevOffsetByType.length; ++i)
    prevOffsetByType[i] = -1;

}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; //导入依赖的package包/类
/**
 * Copied from BufferedDataBlockEncoder. Almost definitely can be improved, but i'm not familiar
 * enough with the concept of the HFileBlockEncodingContext.
 */
@Override
public void encodeKeyValues(ByteBuffer in,
    HFileBlockEncodingContext blkEncodingCtx) throws IOException {
  if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
    throw new IOException(this.getClass().getName() + " only accepts "
        + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context.");
  }

  HFileBlockDefaultEncodingContext encodingCtx
      = (HFileBlockDefaultEncodingContext) blkEncodingCtx;
  encodingCtx.prepareEncoding();
  DataOutputStream dataOut = encodingCtx.getOutputStreamForEncoder();
  internalEncodeKeyValues(dataOut, in, encodingCtx.getHFileContext().isIncludesMvcc(),
      encodingCtx.getHFileContext().isIncludesTags());

  //do i need to check this, or will it always be DataBlockEncoding.PREFIX_TREE?
  if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
    encodingCtx.postEncoding(BlockType.ENCODED_DATA);
  } else {
    encodingCtx.postEncoding(BlockType.DATA);
  }
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; //导入依赖的package包/类
@Override
public void startBlockEncoding(HFileBlockEncodingContext blkEncodingCtx,
    DataOutputStream out) throws IOException {
  if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
    throw new IOException(this.getClass().getName() + " only accepts "
        + HFileBlockDefaultEncodingContext.class.getName() + " as the "
        + "encoding context.");
  }

  HFileBlockDefaultEncodingContext encodingCtx = (HFileBlockDefaultEncodingContext) blkEncodingCtx;
  encodingCtx.prepareEncoding(out);

  NoneEncoder encoder = new NoneEncoder(out, encodingCtx);
  NoneEncodingState state = new NoneEncodingState();
  state.encoder = encoder;
  blkEncodingCtx.setEncodingState(state);
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; //导入依赖的package包/类
/**
 * @param dataBlockEncoder data block encoding algorithm to use
 */
public Writer(HFileDataBlockEncoder dataBlockEncoder, HFileContext fileContext) {
  if (fileContext.getBytesPerChecksum() < HConstants.HFILEBLOCK_HEADER_SIZE) {
    throw new RuntimeException("Unsupported value of bytesPerChecksum. " +
        " Minimum is " + HConstants.HFILEBLOCK_HEADER_SIZE + " but the configured value is " +
        fileContext.getBytesPerChecksum());
  }
  this.dataBlockEncoder = dataBlockEncoder != null?
      dataBlockEncoder: NoOpDataBlockEncoder.INSTANCE;
  this.dataBlockEncodingCtx = this.dataBlockEncoder.
      newDataBlockEncodingContext(HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext);
  // TODO: This should be lazily instantiated since we usually do NOT need this default encoder
  this.defaultBlockEncodingCtx = new HFileBlockDefaultEncodingContext(null,
      HConstants.HFILEBLOCK_DUMMY_HEADER, fileContext);
  // TODO: Set BAOS initial size. Use fileContext.getBlocksize() and add for header/checksum
  baosInMemory = new ByteArrayOutputStream();
  prevOffsetByType = new long[BlockType.values().length];
  for (int i = 0; i < prevOffsetByType.length; ++i) {
    prevOffsetByType[i] = UNSET;
  }
  // TODO: Why fileContext saved away when we have dataBlockEncoder and/or
  // defaultDataBlockEncoder?
  this.fileContext = fileContext;
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; //导入依赖的package包/类
private HFileBlock createBlockOnDisk(List<KeyValue> kvs, HFileBlock block, boolean useTags)
    throws IOException {
  int size;
  HFileBlockEncodingContext context = new HFileBlockDefaultEncodingContext(
      blockEncoder.getDataBlockEncoding(), HConstants.HFILEBLOCK_DUMMY_HEADER,
      block.getHFileContext());

  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  baos.write(block.getDummyHeaderForVersion());
  DataOutputStream dos = new DataOutputStream(baos);
  blockEncoder.startBlockEncoding(context, dos);
  for (KeyValue kv : kvs) {
    blockEncoder.encode(kv, context, dos);
  }
  blockEncoder.endBlockEncoding(context, dos, baos.getBuffer(), BlockType.DATA);
  byte[] encodedBytes = baos.toByteArray();
  size = encodedBytes.length - block.getDummyHeaderForVersion().length;
  return new HFileBlock(context.getBlockType(), size, size, -1, ByteBuffer.wrap(encodedBytes),
      HFileBlock.FILL_HEADER, 0, block.getOnDiskDataSizeWithHeader(), -1,
      block.getHFileContext());
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; //导入依赖的package包/类
/**
 * @param compressionAlgorithm compression algorithm to use
 * @param dataBlockEncoderAlgo data block encoding algorithm to use
 */
public Writer(Compression.Algorithm compressionAlgorithm,
      HFileDataBlockEncoder dataBlockEncoder, boolean includesMemstoreTS) {
  compressAlgo = compressionAlgorithm == null ? NONE : compressionAlgorithm;
  this.dataBlockEncoder = dataBlockEncoder != null
      ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;

  defaultBlockEncodingCtx =
      new HFileBlockDefaultEncodingContext(compressionAlgorithm,
          null, DUMMY_HEADER);
  dataBlockEncodingCtx =
    this.dataBlockEncoder.newDataBlockEncodingContext(
        compressionAlgorithm, DUMMY_HEADER);

  baosInMemory = new ByteArrayOutputStream();

  prevOffsetByType = new long[BlockType.values().length];
  for (int i = 0; i < prevOffsetByType.length; ++i)
    prevOffsetByType[i] = -1;

  this.includesMemstoreTS = includesMemstoreTS;
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; //导入依赖的package包/类
/**
 * Copied from BufferedDataBlockEncoder. Almost definitely can be improved, but i'm not familiar
 * enough with the concept of the HFileBlockEncodingContext.
 */
@Override
public void encodeKeyValues(ByteBuffer in, boolean includesMvccVersion,
    HFileBlockEncodingContext blkEncodingCtx) throws IOException {
  if (blkEncodingCtx.getClass() != HFileBlockDefaultEncodingContext.class) {
    throw new IOException(this.getClass().getName() + " only accepts "
        + HFileBlockDefaultEncodingContext.class.getName() + " as the " + "encoding context.");
  }

  HFileBlockDefaultEncodingContext encodingCtx
      = (HFileBlockDefaultEncodingContext) blkEncodingCtx;
  encodingCtx.prepareEncoding();
  DataOutputStream dataOut = encodingCtx.getOutputStreamForEncoder();
  internalEncodeKeyValues(dataOut, in, includesMvccVersion);

  //do i need to check this, or will it always be DataBlockEncoding.PREFIX_TREE?
  if (encodingCtx.getDataBlockEncoding() != DataBlockEncoding.NONE) {
    encodingCtx.postEncoding(BlockType.ENCODED_DATA);
  } else {
    encodingCtx.postEncoding(BlockType.DATA);
  }
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; //导入依赖的package包/类
/**
 * @param compressionAlgorithm compression algorithm to use
 * @param dataBlockEncoderAlgo data block encoding algorithm to use
 */
public Writer(Compression.Algorithm compressionAlgorithm,
      HFileDataBlockEncoder dataBlockEncoder, boolean includesMemstoreTS) {
  compressAlgo = compressionAlgorithm == null ? NONE : compressionAlgorithm;
  this.dataBlockEncoder = dataBlockEncoder != null
      ? dataBlockEncoder : NoOpDataBlockEncoder.INSTANCE;

  defaultBlockEncodingCtx =
      new HFileBlockDefaultEncodingContext(compressionAlgorithm,
          null, DUMMY_HEADER);
  dataBlockEncodingCtx =
    this.dataBlockEncoder.newOnDiskDataBlockEncodingContext(
        compressionAlgorithm, DUMMY_HEADER);

  baosInMemory = new ByteArrayOutputStream();

  prevOffsetByType = new long[BlockType.values().length];
  for (int i = 0; i < prevOffsetByType.length; ++i)
    prevOffsetByType[i] = -1;

  this.includesMemstoreTS = includesMemstoreTS;
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; //导入依赖的package包/类
@Override
public HFileBlockEncodingContext newDataBlockEncodingContext(
    byte[] dummyHeader, HFileContext fileContext) {
  DataBlockEncoder encoder = encoding.getEncoder();
  if (encoder != null) {
    return encoder.newDataBlockEncodingContext(encoding, dummyHeader, fileContext);
  }
  return new HFileBlockDefaultEncodingContext(null, dummyHeader, fileContext);
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; //导入依赖的package包/类
public Writer(HFileDataBlockEncoder dataBlockEncoder, HFileContext meta) {
  super(dataBlockEncoder, meta);
  compressAlgo = meta.getCompression() == null ? NONE : meta.getCompression();
  this.dataBlockEncoder = dataBlockEncoder != null ? dataBlockEncoder
      : NoOpDataBlockEncoder.INSTANCE;
  defaultBlockEncodingCtx = new HFileBlockDefaultEncodingContext(null, DUMMY_HEADER, meta);
  dataBlockEncodingCtx = this.dataBlockEncoder.newDataBlockEncodingContext(DUMMY_HEADER, meta);
  baosInMemory = new ByteArrayOutputStream();

  prevOffsetByType = new long[BlockType.values().length];
  for (int i = 0; i < prevOffsetByType.length; ++i)
    prevOffsetByType[i] = -1;
}
 

import org.apache.hadoop.hbase.io.encoding.HFileBlockDefaultEncodingContext; //导入依赖的package包/类
@Override
public HFileBlockEncodingContext newDataBlockEncodingContext(
    DataBlockEncoding encoding, byte[] header, HFileContext meta) {
  if(DataBlockEncoding.PREFIX_TREE != encoding){
    //i'm not sure why encoding is in the interface.  Each encoder implementation should probably
    //know it's encoding type
    throw new IllegalArgumentException("only DataBlockEncoding.PREFIX_TREE supported");
  }
  return new HFileBlockDefaultEncodingContext(encoding, header, meta);
}