Java CompactionController类代码示例

import org.apache.cassandra.db.compaction.CompactionController; //导入依赖的package包/类
@Test
public void testNumberOfFiles_abort() throws Exception
{
    testNumberOfFiles_abort(new RewriterTest()
    {
        public void run(ISSTableScanner scanner, CompactionController controller, SSTableReader sstable, ColumnFamilyStore cfs, SSTableRewriter rewriter)
        {
            int files = 1;
            while(scanner.hasNext())
            {
                rewriter.append(new LazilyCompactedRow(controller, Arrays.asList(scanner.next())));
                if (rewriter.currentWriter().getFilePointer() > 25000000)
                {
                    rewriter.switchWriter(getWriter(cfs, sstable.descriptor.directory));
                    files++;
                    assertEquals(cfs.getSSTables().size(), files); // we have one original file plus the ones we have switched out.
                }
            }
            rewriter.abort();
        }
    });
}
 

import org.apache.cassandra.db.compaction.CompactionController; //导入依赖的package包/类
@Test
public void testNumberOfFiles_abort3() throws Exception
{
    testNumberOfFiles_abort(new RewriterTest()
    {
        public void run(ISSTableScanner scanner, CompactionController controller, SSTableReader sstable, ColumnFamilyStore cfs, SSTableRewriter rewriter)
        {
            int files = 1;
            while(scanner.hasNext())
            {
                rewriter.append(new LazilyCompactedRow(controller, Arrays.asList(scanner.next())));
                if (files == 1 && rewriter.currentWriter().getFilePointer() > 10000000)
                {
                    rewriter.switchWriter(getWriter(cfs, sstable.descriptor.directory));
                    files++;
                    assertEquals(cfs.getSSTables().size(), files); // we have one original file plus the ones we have switched out.
                }
            }
            rewriter.abort();
        }
    });
}
 

import org.apache.cassandra.db.compaction.CompactionController; //导入依赖的package包/类
public PurgeStatisticBackend(ColumnFamilyStore cfs, Collection<SSTableReader> sstables, RateLimiter rateLimiter) {
    bytesRead = 0;
    readerQueue = new PriorityQueue<>(sstables.size());
    for (SSTableReader sstable : sstables) {
        length += sstable.uncompressedLength();
        ScannerWrapper scanner = new ScannerWrapper(sstable.descriptor.generation, sstable.getScanner(rateLimiter));
        if (scanner.next()) {
            readerQueue.add(scanner);
        }
    }
    this.controller = new CompactionController(cfs, null, cfs.gcBefore(Util.NOW));
}
 

import org.apache.cassandra.db.compaction.CompactionController; //导入依赖的package包/类
@Test
public void basicTest() throws InterruptedException
{
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    cfs.truncateBlocking();
    for (int j = 0; j < 100; j ++)
    {
        ByteBuffer key = ByteBufferUtil.bytes(String.valueOf(j));
        Mutation rm = new Mutation(KEYSPACE, key);
        rm.add(CF, Util.cellname("0"), ByteBufferUtil.EMPTY_BYTE_BUFFER, j);
        rm.apply();
    }
    cfs.forceBlockingFlush();
    Set<SSTableReader> sstables = new HashSet<>(cfs.getSSTables());
    assertEquals(1, sstables.size());
    SSTableRewriter writer = new SSTableRewriter(cfs, sstables, 1000, false);
    try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategy().getScanners(sstables);)
    {
        ISSTableScanner scanner = scanners.scanners.get(0);
        CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(System.currentTimeMillis()));
        writer.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory));
        while(scanner.hasNext())
        {
            AbstractCompactedRow row = new LazilyCompactedRow(controller, Arrays.asList(scanner.next()));
            writer.append(row);
        }
    }
    Collection<SSTableReader> newsstables = writer.finish();
    cfs.getDataTracker().markCompactedSSTablesReplaced(sstables, newsstables , OperationType.COMPACTION);
    Thread.sleep(100);
    validateCFS(cfs);
    int filecounts = assertFileCounts(sstables.iterator().next().descriptor.directory.list(), 0, 0);
    assertEquals(1, filecounts);

}
 

import org.apache.cassandra.db.compaction.CompactionController; //导入依赖的package包/类
@Test
public void basicTest2() throws InterruptedException
{
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    cfs.truncateBlocking();

    SSTableReader s = writeFile(cfs, 1000);
    cfs.addSSTable(s);
    Set<SSTableReader> sstables = new HashSet<>(cfs.getSSTables());
    assertEquals(1, sstables.size());
    SSTableRewriter.overrideOpenInterval(10000000);
    SSTableRewriter writer = new SSTableRewriter(cfs, sstables, 1000, false);
    try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategy().getScanners(sstables);)
    {
        ISSTableScanner scanner = scanners.scanners.get(0);
        CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(System.currentTimeMillis()));
        writer.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory));
        while (scanner.hasNext())
        {
            AbstractCompactedRow row = new LazilyCompactedRow(controller, Arrays.asList(scanner.next()));
            writer.append(row);
        }
    }
    Collection<SSTableReader> newsstables = writer.finish();
    cfs.getDataTracker().markCompactedSSTablesReplaced(sstables, newsstables, OperationType.COMPACTION);
    Thread.sleep(100);
    validateCFS(cfs);
    int filecounts = assertFileCounts(sstables.iterator().next().descriptor.directory.list(), 0, 0);
    assertEquals(1, filecounts);
}
 

import org.apache.cassandra.db.compaction.CompactionController; //导入依赖的package包/类
@Test
public void testNumberOfFiles_abort2() throws Exception
{
    testNumberOfFiles_abort(new RewriterTest()
    {
        public void run(ISSTableScanner scanner, CompactionController controller, SSTableReader sstable, ColumnFamilyStore cfs, SSTableRewriter rewriter)
        {
            int files = 1;
            while(scanner.hasNext())
            {
                rewriter.append(new LazilyCompactedRow(controller, Arrays.asList(scanner.next())));
                if (rewriter.currentWriter().getFilePointer() > 25000000)
                {
                    rewriter.switchWriter(getWriter(cfs, sstable.descriptor.directory));
                    files++;
                    assertEquals(cfs.getSSTables().size(), files); // we have one original file plus the ones we have switched out.
                }
                if (files == 3)
                {
                    //testing to abort when we have nothing written in the new file
                    rewriter.abort();
                    break;
                }
            }
        }
    });
}
 

import org.apache.cassandra.db.compaction.CompactionController; //导入依赖的package包/类
@Test
public void basicTest() throws InterruptedException
{
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    truncate(cfs);

    for (int j = 0; j < 100; j ++)
    {
        new RowUpdateBuilder(cfs.metadata, j, String.valueOf(j))
            .clustering("0")
            .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
            .build()
            .apply();
    }
    cfs.forceBlockingFlush();
    Set<SSTableReader> sstables = new HashSet<>(cfs.getLiveSSTables());
    assertEquals(1, sstables.size());
    assertEquals(sstables.iterator().next().bytesOnDisk(), cfs.metric.liveDiskSpaceUsed.getCount());
    int nowInSec = FBUtilities.nowInSeconds();
    try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(sstables);
         LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN);
         SSTableRewriter writer = new SSTableRewriter(txn, 1000, false);
         CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(nowInSec));
         CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID()))
    {
        writer.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory, txn));
        while(ci.hasNext())
        {
            writer.append(ci.next());
        }
        writer.finish();
    }
    LifecycleTransaction.waitForDeletions();
    validateCFS(cfs);
    int filecounts = assertFileCounts(sstables.iterator().next().descriptor.directory.list());
    assertEquals(1, filecounts);
    truncate(cfs);
}
 

import org.apache.cassandra.db.compaction.CompactionController; //导入依赖的package包/类
@Test
public void basicTest2() throws InterruptedException
{
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    truncate(cfs);

    SSTableReader s = writeFile(cfs, 1000);
    cfs.addSSTable(s);
    Set<SSTableReader> sstables = new HashSet<>(cfs.getLiveSSTables());
    assertEquals(1, sstables.size());

    int nowInSec = FBUtilities.nowInSeconds();
    try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(sstables);
         LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN);
         SSTableRewriter writer = new SSTableRewriter(txn, 1000, false, 10000000, false);
         CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(nowInSec));
         CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID()))
    {
        writer.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory, txn));
        while (ci.hasNext())
        {
            writer.append(ci.next());
        }
        writer.finish();
    }
    LifecycleTransaction.waitForDeletions();
    validateCFS(cfs);
    int filecounts = assertFileCounts(sstables.iterator().next().descriptor.directory.list());
    assertEquals(1, filecounts);
}
 

import org.apache.cassandra.db.compaction.CompactionController; //导入依赖的package包/类
@Test
public void testNumberOfFiles_abort() throws Exception
{
    testNumberOfFiles_abort(new RewriterTest()
    {
        public void run(ISSTableScanner scanner,
                        CompactionController controller,
                        SSTableReader sstable,
                        ColumnFamilyStore cfs,
                        SSTableRewriter rewriter,
                        LifecycleTransaction txn)
        {
            try (CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID()))
            {
                int files = 1;
                while (ci.hasNext())
                {
                    rewriter.append(ci.next());
                    if (rewriter.currentWriter().getFilePointer() > 25000000)
                    {
                    rewriter.switchWriter(getWriter(cfs, sstable.descriptor.directory, txn));
                        files++;
                        assertEquals(cfs.getLiveSSTables().size(), files); // we have one original file plus the ones we have switched out.
                    }
                }
                rewriter.abort();
            }
        }
    });
}
 

import org.apache.cassandra.db.compaction.CompactionController; //导入依赖的package包/类
@Test
public void testNumberOfFiles_abort2() throws Exception
{
    testNumberOfFiles_abort(new RewriterTest()
    {
        public void run(ISSTableScanner scanner,
                        CompactionController controller,
                        SSTableReader sstable,
                        ColumnFamilyStore cfs,
                        SSTableRewriter rewriter,
                        LifecycleTransaction txn)
        {
            try (CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID()))
            {
                int files = 1;
                while (ci.hasNext())
                {
                    rewriter.append(ci.next());
                    if (rewriter.currentWriter().getFilePointer() > 25000000)
                    {
                    rewriter.switchWriter(getWriter(cfs, sstable.descriptor.directory, txn));
                        files++;
                        assertEquals(cfs.getLiveSSTables().size(), files); // we have one original file plus the ones we have switched out.
                    }
                    if (files == 3)
                    {
                        //testing to abort when we have nothing written in the new file
                        rewriter.abort();
                        break;
                    }
                }
            }
        }
    });
}
 

import org.apache.cassandra.db.compaction.CompactionController; //导入依赖的package包/类
@Test
public void testNumberOfFiles_abort3() throws Exception
{
    testNumberOfFiles_abort(new RewriterTest()
    {
        public void run(ISSTableScanner scanner,
                        CompactionController controller,
                        SSTableReader sstable,
                        ColumnFamilyStore cfs,
                        SSTableRewriter rewriter,
                        LifecycleTransaction txn)
        {
            try(CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID()))
            {
                int files = 1;
                while (ci.hasNext())
                {
                    rewriter.append(ci.next());
                    if (files == 1 && rewriter.currentWriter().getFilePointer() > 10000000)
                    {
                    rewriter.switchWriter(getWriter(cfs, sstable.descriptor.directory, txn));
                        files++;
                        assertEquals(cfs.getLiveSSTables().size(), files); // we have one original file plus the ones we have switched out.
                    }
                }
                rewriter.abort();
            }
        }
    });
}
 

import org.apache.cassandra.db.compaction.CompactionController; //导入依赖的package包/类
@Test
public void testNumberOfFiles_truncate() throws Exception
{
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    truncate(cfs);
    cfs.disableAutoCompaction();

    SSTableReader s = writeFile(cfs, 1000);
    cfs.addSSTable(s);
    Set<SSTableReader> compacting = Sets.newHashSet(s);

    List<SSTableReader> sstables;
    int files = 1;
    try (ISSTableScanner scanner = s.getScanner();
         CompactionController controller = new CompactionController(cfs, compacting, 0);
         LifecycleTransaction txn = cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN);
         SSTableRewriter rewriter = new SSTableRewriter(txn, 1000, false, 10000000, false);
         CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID()))
    {
        rewriter.switchWriter(getWriter(cfs, s.descriptor.directory, txn));
        while(ci.hasNext())
        {
            rewriter.append(ci.next());
            if (rewriter.currentWriter().getOnDiskFilePointer() > 25000000)
            {
                rewriter.switchWriter(getWriter(cfs, s.descriptor.directory, txn));
                files++;
                assertEquals(cfs.getLiveSSTables().size(), files); // we have one original file plus the ones we have switched out.
            }
        }

        sstables = rewriter.finish();
    }

    LifecycleTransaction.waitForDeletions();
    assertFileCounts(s.descriptor.directory.list());
    validateCFS(cfs);
}
 

import org.apache.cassandra.db.compaction.CompactionController; //导入依赖的package包/类
@Test
public void testCanonicalView() throws Exception
{
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    truncate(cfs);

    SSTableReader s = writeFile(cfs, 1000);
    cfs.addSSTable(s);
    Set<SSTableReader> sstables = Sets.newHashSet(s);
    assertEquals(1, sstables.size());
    boolean checked = false;
    try (ISSTableScanner scanner = sstables.iterator().next().getScanner();
         CompactionController controller = new CompactionController(cfs, sstables, 0);
         LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN);
         SSTableRewriter writer = new SSTableRewriter(txn, 1000, false, 10000000, false);
         CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID())
    )
    {
        writer.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory, txn));
        while (ci.hasNext())
        {
            writer.append(ci.next());
            if (!checked && writer.currentWriter().getFilePointer() > 15000000)
            {
                checked = true;
                ColumnFamilyStore.ViewFragment viewFragment = cfs.select(View.selectFunction(SSTableSet.CANONICAL));
                // canonical view should have only one SSTable which is not opened early.
                assertEquals(1, viewFragment.sstables.size());
                SSTableReader sstable = viewFragment.sstables.get(0);
                assertEquals(s.descriptor, sstable.descriptor);
                assertTrue("Found early opened SSTable in canonical view: " + sstable.getFilename(), sstable.openReason != SSTableReader.OpenReason.EARLY);
            }
        }
    }
    truncateCF();
    validateCFS(cfs);
}
 

import org.apache.cassandra.db.compaction.CompactionController; //导入依赖的package包/类
/**
 * emulates anticompaction - writing from one source sstable to two new sstables
 *
 * @throws IOException
 */
@Test
public void testTwoWriters() throws IOException
{
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    truncate(cfs);

    SSTableReader s = writeFile(cfs, 1000);
    cfs.addSSTable(s);
    Set<SSTableReader> sstables = Sets.newHashSet(s);
    assertEquals(1, sstables.size());
    int nowInSec = FBUtilities.nowInSeconds();
    try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(sstables);
         LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN);
         SSTableRewriter writer = new SSTableRewriter(txn, 1000, false, false);
         SSTableRewriter writer2 = new SSTableRewriter(txn, 1000, false, false);
         CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(nowInSec));
         CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID())
         )
    {
        writer.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory, txn));
        writer2.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory, txn));
        while (ci.hasNext())
        {
            if (writer.currentWriter().getFilePointer() < 15000000)
                writer.append(ci.next());
            else
                writer2.append(ci.next());
        }
        for (int i = 0; i < 5000; i++)
            assertFalse(Util.getOnlyPartition(Util.cmd(cfs, ByteBufferUtil.bytes(i)).build()).isEmpty());
    }
    truncateCF();
    validateCFS(cfs);
}
 

import org.apache.cassandra.db.compaction.CompactionController; //导入依赖的package包/类
public EchoedRow(CompactionController controller, SSTableIdentityIterator row)
{
    super(row.getKey());
    this.row = row;
    this.gcBefore = controller.gcBefore;
    // Reset SSTableIdentityIterator because we have not guarantee the filePointer hasn't moved since the Iterator was built
    row.reset();
}