Java CompressionMetadata.Chunk方法代码示例

import org.apache.cassandra.io.compress.CompressionMetadata; //导入方法依赖的package包/类
protected void runMayThrow() throws Exception
{
    byte[] compressedWithCRC;
    while (chunks.hasNext())
    {
        CompressionMetadata.Chunk chunk = chunks.next();

        int readLength = chunk.length + 4; // read with CRC
        compressedWithCRC = new byte[readLength];

        int bufferRead = 0;
        while (bufferRead < readLength)
        {
            int r = source.read(compressedWithCRC, bufferRead, readLength - bufferRead);
            if (r < 0)
            {
                dataBuffer.put(POISON_PILL);
                return; // throw exception where we consume dataBuffer
            }
            bufferRead += r;
        }
        dataBuffer.put(compressedWithCRC);
    }
}
 

import org.apache.cassandra.io.compress.CompressionMetadata; //导入方法依赖的package包/类
protected void runMayThrow() throws Exception
{
    byte[] compressedWithCRC;
    while (chunks.hasNext())
    {
        CompressionMetadata.Chunk chunk = chunks.next();

        int readLength = chunk.length + 4; // read with CRC
        compressedWithCRC = new byte[readLength];

        int bufferRead = 0;
        while (bufferRead < readLength)
            bufferRead += source.read(compressedWithCRC, bufferRead, readLength - bufferRead);
        dataBuffer.put(compressedWithCRC);
    }
}
 

import org.apache.cassandra.io.compress.CompressionMetadata; //导入方法依赖的package包/类
private long calculateSize()
{
    long transferSize = 0;
    if (compressionInfo != null)
    {
        // calculate total length of transferring chunks
        for (CompressionMetadata.Chunk chunk : compressionInfo.chunks)
            transferSize += chunk.length + 4; // 4 bytes for CRC
    }
    else if (compressionMetadata != null)
    {
        transferSize = compressionMetadata.getTotalSizeForSections(sections);
    }
    else
    {
        for (Pair<Long, Long> section : sections)
            transferSize += section.right - section.left;
    }
    return transferSize;
}
 

import org.apache.cassandra.io.compress.CompressionMetadata; //导入方法依赖的package包/类
public CompressionInfo deserialize(DataInput in, int version) throws IOException
{
    // chunks
    int chunkCount = in.readInt();
    if (chunkCount < 0)
        return null;

    CompressionMetadata.Chunk[] chunks = new CompressionMetadata.Chunk[chunkCount];
    for (int i = 0; i < chunkCount; i++)
        chunks[i] = CompressionMetadata.Chunk.serializer.deserialize(in, version);

    // compression params
    CompressionParameters parameters = CompressionParameters.serializer.deserialize(in, version);
    return new CompressionInfo(chunks, parameters);
}
 

import org.apache.cassandra.io.compress.CompressionMetadata; //导入方法依赖的package包/类
@Override
protected long totalSize()
{
    long size = 0;
    // calculate total length of transferring chunks
    for (CompressionMetadata.Chunk chunk : compressionInfo.chunks)
        size += chunk.length + 4; // 4 bytes for CRC
    return size;
}
 

import org.apache.cassandra.io.compress.CompressionMetadata; //导入方法依赖的package包/类
private List<Pair<Long, Long>> getTransferSections(CompressionMetadata.Chunk[] chunks)
{
    List<Pair<Long, Long>> transferSections = new ArrayList<>();
    Pair<Long, Long> lastSection = null;
    for (CompressionMetadata.Chunk chunk : chunks)
    {
        if (lastSection != null)
        {
            if (chunk.offset == lastSection.right)
            {
                // extend previous section to end of this chunk
                lastSection = Pair.create(lastSection.left, chunk.offset + chunk.length + 4); // 4 bytes for CRC
            }
            else
            {
                transferSections.add(lastSection);
                lastSection = Pair.create(chunk.offset, chunk.offset + chunk.length + 4);
            }
        }
        else
        {
            lastSection = Pair.create(chunk.offset, chunk.offset + chunk.length + 4);
        }
    }
    if (lastSection != null)
        transferSections.add(lastSection);
    return transferSections;
}
 

import org.apache.cassandra.io.compress.CompressionMetadata; //导入方法依赖的package包/类
public CompressionInfo deserialize(DataInputPlus in, int version) throws IOException
{
    // chunks
    int chunkCount = in.readInt();
    if (chunkCount < 0)
        return null;

    CompressionMetadata.Chunk[] chunks = new CompressionMetadata.Chunk[chunkCount];
    for (int i = 0; i < chunkCount; i++)
        chunks[i] = CompressionMetadata.Chunk.serializer.deserialize(in, version);

    // compression params
    CompressionParams parameters = CompressionParams.serializer.deserialize(in, version);
    return new CompressionInfo(chunks, parameters);
}
 

import org.apache.cassandra.io.compress.CompressionMetadata; //导入方法依赖的package包/类
protected void runMayThrow() throws Exception
{
    byte[] compressedWithCRC;
    while (chunks.hasNext())
    {
        CompressionMetadata.Chunk chunk = chunks.next();

        int readLength = chunk.length + 4; // read with CRC
        compressedWithCRC = new byte[readLength];

        int bufferRead = 0;
        while (bufferRead < readLength)
        {
            try
            {
                int r = source.read(compressedWithCRC, bufferRead, readLength - bufferRead);
                if (r < 0)
                {
                    readException = new EOFException("No chunk available");
                    dataBuffer.put(POISON_PILL);
                    return; // throw exception where we consume dataBuffer
                }
                bufferRead += r;
            }
            catch (IOException e)
            {
                logger.warn("Error while reading compressed input stream.", e);
                readException = e;
                dataBuffer.put(POISON_PILL);
                return; // throw exception where we consume dataBuffer
            }
        }
        dataBuffer.put(compressedWithCRC);
    }
}
 

import org.apache.cassandra.io.compress.CompressionMetadata; //导入方法依赖的package包/类
private void overrideWithGarbage(SSTableReader sstable, ByteBuffer key1, ByteBuffer key2) throws IOException
{
    boolean compression = Boolean.parseBoolean(System.getProperty("cassandra.test.compression", "false"));
    long startPosition, endPosition;

    if (compression)
    { // overwrite with garbage the compression chunks from key1 to key2
        CompressionMetadata compData = CompressionMetadata.create(sstable.getFilename());

        CompressionMetadata.Chunk chunk1 = compData.chunkFor(
                sstable.getPosition(PartitionPosition.ForKey.get(key1, sstable.getPartitioner()), SSTableReader.Operator.EQ).position);
        CompressionMetadata.Chunk chunk2 = compData.chunkFor(
                sstable.getPosition(PartitionPosition.ForKey.get(key2, sstable.getPartitioner()), SSTableReader.Operator.EQ).position);

        startPosition = Math.min(chunk1.offset, chunk2.offset);
        endPosition = Math.max(chunk1.offset + chunk1.length, chunk2.offset + chunk2.length);

        compData.close();
    }
    else
    { // overwrite with garbage from key1 to key2
        long row0Start = sstable.getPosition(PartitionPosition.ForKey.get(key1, sstable.getPartitioner()), SSTableReader.Operator.EQ).position;
        long row1Start = sstable.getPosition(PartitionPosition.ForKey.get(key2, sstable.getPartitioner()), SSTableReader.Operator.EQ).position;
        startPosition = Math.min(row0Start, row1Start);
        endPosition = Math.max(row0Start, row1Start);
    }

    overrideWithGarbage(sstable, startPosition, endPosition);
}
 

import org.apache.cassandra.io.compress.CompressionMetadata; //导入方法依赖的package包/类
private List<Pair<Long, Long>> getTransferSections(CompressionMetadata.Chunk[] chunks)
{
    List<Pair<Long, Long>> transferSections = new ArrayList<Pair<Long, Long>>();
    Pair<Long, Long> lastSection = null;
    for (CompressionMetadata.Chunk chunk : chunks)
    {
        if (lastSection != null)
        {
            if (chunk.offset == lastSection.right)
            {
                // extend previous section to end of this chunk
                lastSection = Pair.create(lastSection.left, chunk.offset + chunk.length + 4); // 4 bytes for CRC
            }
            else
            {
                transferSections.add(lastSection);
                lastSection = Pair.create(chunk.offset, chunk.offset + chunk.length + 4);
            }
        }
        else
        {
            lastSection = Pair.create(chunk.offset, chunk.offset + chunk.length + 4);
        }
    }
    if (lastSection != null)
        transferSections.add(lastSection);
    return transferSections;
}
 

import org.apache.cassandra.io.compress.CompressionMetadata; //导入方法依赖的package包/类
public PendingFile(SSTableReader sstable,
                   Descriptor desc,
                   String component,
                   List<Pair<Long,Long>> sections,
                   OperationType type,
                   long estimatedKeys,
                   CompressionInfo compressionInfo)
{
    this.sstable = sstable;
    this.desc = desc;
    this.component = component;
    this.sections = sections;
    this.type = type;

    long tempSize = 0;
    if (compressionInfo == null)
    {
        for (Pair<Long, Long> section : sections)
            tempSize += section.right - section.left;
    }
    else
    {
        // calculate total length of transferring chunks
        for (CompressionMetadata.Chunk chunk : compressionInfo.chunks)
            tempSize += chunk.length + 4; // 4 bytes for CRC
    }
    size = tempSize;

    this.estimatedKeys = estimatedKeys;
    this.compressionInfo = compressionInfo;
}
 

import org.apache.cassandra.io.compress.CompressionMetadata; //导入方法依赖的package包/类
public CompressionInfo(CompressionMetadata.Chunk[] chunks, CompressionParameters parameters)
{
    assert chunks != null && parameters != null;
    this.chunks = chunks;
    this.parameters = parameters;
}
 

import org.apache.cassandra.io.compress.CompressionMetadata; //导入方法依赖的package包/类
@Test
public void testMapForCompressionMetadata() throws Exception
{
    int OLD_MAX_SEGMENT_SIZE = MmappedRegions.MAX_SEGMENT_SIZE;
    MmappedRegions.MAX_SEGMENT_SIZE = 1024;

    ByteBuffer buffer = allocateBuffer(128 * 1024);
    File f = File.createTempFile("testMapForCompressionMetadata", "1");
    f.deleteOnExit();

    File cf = File.createTempFile(f.getName() + ".metadata", "1");
    cf.deleteOnExit();

    MetadataCollector sstableMetadataCollector = new MetadataCollector(new ClusteringComparator(BytesType.instance));
    try(SequentialWriter writer = new CompressedSequentialWriter(f,
                                                                 cf.getAbsolutePath(),
                                                                 CompressionParams.snappy(),
                                                                 sstableMetadataCollector))
    {
        writer.write(buffer);
        writer.finish();
    }

    CompressionMetadata metadata = new CompressionMetadata(cf.getAbsolutePath(), f.length(), ChecksumType.CRC32);
    try(ChannelProxy channel = new ChannelProxy(f);
        MmappedRegions regions = MmappedRegions.map(channel, metadata))
    {

        assertFalse(regions.isEmpty());
        int i = 0;
        while(i < buffer.capacity())
        {
            CompressionMetadata.Chunk chunk = metadata.chunkFor(i);

            MmappedRegions.Region region = regions.floor(chunk.offset);
            assertNotNull(region);

            ByteBuffer compressedChunk = region.buffer.duplicate();
            assertNotNull(compressedChunk);
            assertEquals(chunk.length + 4, compressedChunk.capacity());

            assertEquals(chunk.offset, region.bottom());
            assertEquals(chunk.offset + chunk.length + 4, region.top());

            i += metadata.chunkLength();
        }
    }
    finally
    {
        MmappedRegions.MAX_SEGMENT_SIZE = OLD_MAX_SEGMENT_SIZE;
        metadata.close();
    }
}
 

import org.apache.cassandra.io.compress.CompressionMetadata; //导入方法依赖的package包/类
public CompressionInfo(CompressionMetadata.Chunk[] chunks, CompressionParams parameters)
{
    assert chunks != null && parameters != null;
    this.chunks = chunks;
    this.parameters = parameters;
}