Java DataBlockEncoder类代码示例

import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; //导入依赖的package包/类
private ByteBuffer encodeBufferToHFileBlockBuffer(ByteBuffer in,
    DataBlockEncoding algo, boolean includesMemstoreTS,
    byte[] dummyHeader) {
  ByteArrayOutputStream encodedStream = new ByteArrayOutputStream();
  DataOutputStream dataOut = new DataOutputStream(encodedStream);
  DataBlockEncoder encoder = algo.getEncoder();
  try {
    encodedStream.write(dummyHeader);
    algo.writeIdInBytes(dataOut);
    encoder.compressKeyValues(dataOut, in,
        includesMemstoreTS);
  } catch (IOException e) {
    throw new RuntimeException(String.format("Bug in data block encoder " +
        "'%s', it probably requested too much data", algo.toString()), e);
  }
  return ByteBuffer.wrap(encodedStream.toByteArray());
}
 

import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; //导入依赖的package包/类
/**
 * Updates the current block to be the given {@link HFileBlock}. Seeks to
 * the the first key/value pair.
 *
 * @param newBlock the block to make current
 */
private void updateCurrentBlock(HFileBlock newBlock) {
  block = newBlock;

  // sanity checks
  if (block.getBlockType() != BlockType.ENCODED_DATA) {
    throw new IllegalStateException(
        "EncodedScannerV2 works only on encoded data blocks");
  }

  short dataBlockEncoderId = block.getDataBlockEncodingId();
  if (dataBlockEncoder == null ||
      !DataBlockEncoding.isCorrectEncoder(dataBlockEncoder,
          dataBlockEncoderId)) {
    DataBlockEncoder encoder =
        DataBlockEncoding.getDataBlockEncoderById(dataBlockEncoderId);
    setDataBlockEncoder(encoder);
  }

  seeker.setCurrentBuffer(getEncodedBuffer(newBlock));
  blockFetches++;

  // Reset the next indexed key
  this.nextIndexedKey = null;
}
 

import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; //导入依赖的package包/类
/**
 * Updates the current block to be the given {@link HFileBlock}. Seeks to
 * the the first key/value pair.
 *
 * @param newBlock the block to make current
 */
private void updateCurrentBlock(HFileBlock newBlock) {
  block = newBlock;

  // sanity checks
  if (block.getBlockType() != BlockType.ENCODED_DATA) {
    throw new IllegalStateException(
        "EncodedScannerV2 works only on encoded data blocks");
  }

  short dataBlockEncoderId = block.getDataBlockEncodingId();
  if (dataBlockEncoder == null ||
      !DataBlockEncoding.isCorrectEncoder(dataBlockEncoder,
          dataBlockEncoderId)) {
    DataBlockEncoder encoder =
        DataBlockEncoding.getDataBlockEncoderById(dataBlockEncoderId);
    setDataBlockEncoder(encoder);
  }

  seeker.setCurrentBuffer(getEncodedBuffer(newBlock));
  blockFetches++;
}
 

import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; //导入依赖的package包/类
@Override
public HFileBlockEncodingContext newDataBlockEncodingContext(
    byte[] dummyHeader, HFileContext fileContext) {
  DataBlockEncoder encoder = encoding.getEncoder();
  if (encoder != null) {
    return encoder.newDataBlockEncodingContext(encoding, dummyHeader, fileContext);
  }
  return new HFileBlockDefaultEncodingContext(null, dummyHeader, fileContext);
}
 

import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; //导入依赖的package包/类
@Override
public HFileBlockDecodingContext newDataBlockDecodingContext(HFileContext fileContext) {
  DataBlockEncoder encoder = encoding.getEncoder();
  if (encoder != null) {
    return encoder.newDataBlockDecodingContext(fileContext);
  }
  return new HFileBlockDefaultDecodingContext(fileContext);
}
 

import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; //导入依赖的package包/类
/**
 * Encode a block of key value pairs.
 *
 * @param in input data to encode
 * @param algo encoding algorithm
 * @param encodeCtx where will the output data be stored
 */
private void encodeBufferToHFileBlockBuffer(ByteBuffer in, DataBlockEncoding algo,
    HFileBlockEncodingContext encodeCtx) {
  DataBlockEncoder encoder = algo.getEncoder();
  try {
    encoder.encodeKeyValues(in, encodeCtx);
  } catch (IOException e) {
    throw new RuntimeException(String.format(
        "Bug in data block encoder "
            + "'%s', it probably requested too much data, " +
            "exception message: %s.",
            algo.toString(), e.getMessage()), e);
  }
}
 

import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; //导入依赖的package包/类
/**
 * Encode a block of key value pairs.
 *
 * @param in input data to encode
 * @param algo encoding algorithm
 * @param includesMemstoreTS includes memstore timestamp or not
 * @param encodeCtx where will the output data be stored
 */
private void encodeBufferToHFileBlockBuffer(ByteBuffer in,
    DataBlockEncoding algo, boolean includesMemstoreTS,
    HFileBlockEncodingContext encodeCtx) {
  DataBlockEncoder encoder = algo.getEncoder();
  try {
    encoder.encodeKeyValues(in, includesMemstoreTS, encodeCtx);
  } catch (IOException e) {
    throw new RuntimeException(String.format(
        "Bug in data block encoder "
            + "'%s', it probably requested too much data, " +
            "exception message: %s.",
            algo.toString(), e.getMessage()), e);
  }
}
 

import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; //导入依赖的package包/类
@Override
public HFileBlockEncodingContext newDataBlockEncodingContext(
    Algorithm compressionAlgorithm,  byte[] dummyHeader) {
  DataBlockEncoder encoder = encoding.getEncoder();
  if (encoder != null) {
    return encoder.newDataBlockEncodingContext(
      compressionAlgorithm, encoding, dummyHeader);
  }
  return new HFileBlockDefaultEncodingContext(
    compressionAlgorithm, null, dummyHeader);
}
 

import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; //导入依赖的package包/类
@Override
public HFileBlockDecodingContext newDataBlockDecodingContext(
    Algorithm compressionAlgorithm) {
  DataBlockEncoder encoder = encoding.getEncoder();
  if (encoder != null) {
    return encoder.newDataBlockDecodingContext(compressionAlgorithm);
  }
  return new HFileBlockDefaultDecodingContext(compressionAlgorithm);
}
 

import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; //导入依赖的package包/类
@Override
public HFileBlockEncodingContext newOnDiskDataBlockEncodingContext(
    Algorithm compressionAlgorithm,  byte[] dummyHeader) {
  if (onDisk != null) {
    DataBlockEncoder encoder = onDisk.getEncoder();
    if (encoder != null) {
      return encoder.newDataBlockEncodingContext(
          compressionAlgorithm, onDisk, dummyHeader);
    }
  }
  return new HFileBlockDefaultEncodingContext(compressionAlgorithm,
      null, dummyHeader);
}
 

import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; //导入依赖的package包/类
@Override
public HFileBlockDecodingContext newOnDiskDataBlockDecodingContext(
    Algorithm compressionAlgorithm) {
  if (onDisk != null) {
    DataBlockEncoder encoder = onDisk.getEncoder();
    if (encoder != null) {
      return encoder.newDataBlockDecodingContext(
          compressionAlgorithm);
    }
  }
  return new HFileBlockDefaultDecodingContext(compressionAlgorithm);
}
 

import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; //导入依赖的package包/类
/**
 * Check statistics for given HFile for different data block encoders.
 * @param scanner Of file which will be compressed.
 * @param kvLimit Maximal count of KeyValue which will be processed.
 * @throws IOException thrown if scanner is invalid
 */
public void checkStatistics(final KeyValueScanner scanner, final int kvLimit)
    throws IOException {
  scanner.seek(KeyValue.LOWESTKEY);

  KeyValue currentKV;

  byte[] previousKey = null;
  byte[] currentKey;

  DataBlockEncoding[] encodings = DataBlockEncoding.values();

  ByteArrayOutputStream uncompressedOutputStream =
      new ByteArrayOutputStream();

  int j = 0;
  while ((currentKV = KeyValueUtil.ensureKeyValue(scanner.next())) != null && j < kvLimit) {
    // Iterates through key/value pairs
    j++;
    currentKey = currentKV.getKey();
    if (previousKey != null) {
      for (int i = 0; i < previousKey.length && i < currentKey.length &&
          previousKey[i] == currentKey[i]; ++i) {
        totalKeyRedundancyLength++;
      }
    }

    uncompressedOutputStream.write(currentKV.getBuffer(),
        currentKV.getOffset(), currentKV.getLength());

    previousKey = currentKey;

    int kLen = currentKV.getKeyLength();
    int vLen = currentKV.getValueLength();
    int cfLen = currentKV.getFamilyLength(currentKV.getFamilyOffset());
    int restLen = currentKV.getLength() - kLen - vLen;

    totalKeyLength += kLen;
    totalValueLength += vLen;
    totalPrefixLength += restLen;
    totalCFLength += cfLen;
  }

  rawKVs = uncompressedOutputStream.toByteArray();
  boolean useTag = (currentKV.getTagsLength() > 0);
  for (DataBlockEncoding encoding : encodings) {
    if (encoding == DataBlockEncoding.NONE) {
      continue;
    }
    DataBlockEncoder d = encoding.getEncoder();
    HFileContext meta = new HFileContextBuilder()
                        .withCompression(Compression.Algorithm.NONE)
                        .withIncludesMvcc(includesMemstoreTS)
                        .withIncludesTags(useTag).build();
    codecs.add(new EncodedDataBlock(d, encoding, rawKVs, meta ));
  }
}
 

import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; //导入依赖的package包/类
private void setDataBlockEncoder(DataBlockEncoder dataBlockEncoder) {
  this.dataBlockEncoder = dataBlockEncoder;
  seeker = dataBlockEncoder.createSeeker(reader.getComparator(),
      includesMemstoreTS);
}
 

import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; //导入依赖的package包/类
/**
 * Check statistics for given HFile for different data block encoders.
 * @param scanner Of file which will be compressed.
 * @param kvLimit Maximal count of KeyValue which will be processed.
 * @throws IOException thrown if scanner is invalid
 */
public void checkStatistics(final KeyValueScanner scanner, final int kvLimit)
    throws IOException {
  scanner.seek(KeyValue.LOWESTKEY);

  KeyValue currentKv;

  byte[] previousKey = null;
  byte[] currentKey;

  List<DataBlockEncoder> dataBlockEncoders =
      DataBlockEncoding.getAllEncoders();

  for (DataBlockEncoder d : dataBlockEncoders) {
    codecs.add(new EncodedDataBlock(d, includesMemstoreTS));
  }

  int j = 0;
  while ((currentKv = scanner.next()) != null && j < kvLimit) {
    // Iterates through key/value pairs
    j++;
    currentKey = currentKv.getKey();
    if (previousKey != null) {
      for (int i = 0; i < previousKey.length && i < currentKey.length &&
          previousKey[i] == currentKey[i]; ++i) {
        totalKeyRedundancyLength++;
      }
    }

    for (EncodedDataBlock codec : codecs) {
      codec.addKv(currentKv);
    }

    previousKey = currentKey;

    totalPrefixLength += currentKv.getLength() - currentKv.getKeyLength() -
        currentKv.getValueLength();
    totalKeyLength += currentKv.getKeyLength();
    totalValueLength += currentKv.getValueLength();
  }
}
 

import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; //导入依赖的package包/类
/**
 * Check statistics for given HFile for different data block encoders.
 * @param scanner Of file which will be compressed.
 * @param kvLimit Maximal count of KeyValue which will be processed.
 * @throws IOException thrown if scanner is invalid
 */
public void checkStatistics(final KeyValueScanner scanner, final int kvLimit)
    throws IOException {
  scanner.seek(KeyValue.LOWESTKEY);

  KeyValue currentKV;

  byte[] previousKey = null;
  byte[] currentKey;

  DataBlockEncoding[] encodings = DataBlockEncoding.values();

  ByteArrayOutputStream uncompressedOutputStream =
      new ByteArrayOutputStream();

  int j = 0;
  while ((currentKV = scanner.next()) != null && j < kvLimit) {
    // Iterates through key/value pairs
    j++;
    currentKey = currentKV.getKey();
    if (previousKey != null) {
      for (int i = 0; i < previousKey.length && i < currentKey.length &&
          previousKey[i] == currentKey[i]; ++i) {
        totalKeyRedundancyLength++;
      }
    }

    uncompressedOutputStream.write(currentKV.getBuffer(),
        currentKV.getOffset(), currentKV.getLength());

    previousKey = currentKey;

    int kLen = currentKV.getKeyLength();
    int vLen = currentKV.getValueLength();
    int cfLen = currentKV.getFamilyLength(currentKV.getFamilyOffset());
    int restLen = currentKV.getLength() - kLen - vLen;

    totalKeyLength += kLen;
    totalValueLength += vLen;
    totalPrefixLength += restLen;
    totalCFLength += cfLen;
  }

  rawKVs = uncompressedOutputStream.toByteArray();
  boolean useTag = (currentKV.getTagsLength() > 0);
  for (DataBlockEncoding encoding : encodings) {
    if (encoding == DataBlockEncoding.NONE) {
      continue;
    }
    DataBlockEncoder d = encoding.getEncoder();
    HFileContext meta = new HFileContextBuilder()
                        .withCompression(Compression.Algorithm.NONE)
                        .withIncludesMvcc(includesMemstoreTS)
                        .withIncludesTags(useTag).build();
    codecs.add(new EncodedDataBlock(d, encoding, rawKVs, meta ));
  }
}