Java OperationType.COMPACTION属性代码示例

@Test
public void basicTest() throws InterruptedException
{
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    truncate(cfs);

    for (int j = 0; j < 100; j ++)
    {
        new RowUpdateBuilder(cfs.metadata, j, String.valueOf(j))
            .clustering("0")
            .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
            .build()
            .apply();
    }
    cfs.forceBlockingFlush();
    Set<SSTableReader> sstables = new HashSet<>(cfs.getLiveSSTables());
    assertEquals(1, sstables.size());
    assertEquals(sstables.iterator().next().bytesOnDisk(), cfs.metric.liveDiskSpaceUsed.getCount());
    int nowInSec = FBUtilities.nowInSeconds();
    try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(sstables);
         LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN);
         SSTableRewriter writer = new SSTableRewriter(txn, 1000, false);
         CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(nowInSec));
         CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID()))
    {
        writer.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory, txn));
        while(ci.hasNext())
        {
            writer.append(ci.next());
        }
        writer.finish();
    }
    LifecycleTransaction.waitForDeletions();
    validateCFS(cfs);
    int filecounts = assertFileCounts(sstables.iterator().next().descriptor.directory.list());
    assertEquals(1, filecounts);
    truncate(cfs);
}
 

@Test
public void basicTest2() throws InterruptedException
{
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    truncate(cfs);

    SSTableReader s = writeFile(cfs, 1000);
    cfs.addSSTable(s);
    Set<SSTableReader> sstables = new HashSet<>(cfs.getLiveSSTables());
    assertEquals(1, sstables.size());

    int nowInSec = FBUtilities.nowInSeconds();
    try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(sstables);
         LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN);
         SSTableRewriter writer = new SSTableRewriter(txn, 1000, false, 10000000, false);
         CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(nowInSec));
         CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID()))
    {
        writer.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory, txn));
        while (ci.hasNext())
        {
            writer.append(ci.next());
        }
        writer.finish();
    }
    LifecycleTransaction.waitForDeletions();
    validateCFS(cfs);
    int filecounts = assertFileCounts(sstables.iterator().next().descriptor.directory.list());
    assertEquals(1, filecounts);
}
 

@Test
public void testNumberOfFiles_truncate() throws Exception
{
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    truncate(cfs);
    cfs.disableAutoCompaction();

    SSTableReader s = writeFile(cfs, 1000);
    cfs.addSSTable(s);
    Set<SSTableReader> compacting = Sets.newHashSet(s);

    List<SSTableReader> sstables;
    int files = 1;
    try (ISSTableScanner scanner = s.getScanner();
         CompactionController controller = new CompactionController(cfs, compacting, 0);
         LifecycleTransaction txn = cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN);
         SSTableRewriter rewriter = new SSTableRewriter(txn, 1000, false, 10000000, false);
         CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID()))
    {
        rewriter.switchWriter(getWriter(cfs, s.descriptor.directory, txn));
        while(ci.hasNext())
        {
            rewriter.append(ci.next());
            if (rewriter.currentWriter().getOnDiskFilePointer() > 25000000)
            {
                rewriter.switchWriter(getWriter(cfs, s.descriptor.directory, txn));
                files++;
                assertEquals(cfs.getLiveSSTables().size(), files); // we have one original file plus the ones we have switched out.
            }
        }

        sstables = rewriter.finish();
    }

    LifecycleTransaction.waitForDeletions();
    assertFileCounts(s.descriptor.directory.list());
    validateCFS(cfs);
}
 

@Test
public void testCanonicalView() throws Exception
{
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    truncate(cfs);

    SSTableReader s = writeFile(cfs, 1000);
    cfs.addSSTable(s);
    Set<SSTableReader> sstables = Sets.newHashSet(s);
    assertEquals(1, sstables.size());
    boolean checked = false;
    try (ISSTableScanner scanner = sstables.iterator().next().getScanner();
         CompactionController controller = new CompactionController(cfs, sstables, 0);
         LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN);
         SSTableRewriter writer = new SSTableRewriter(txn, 1000, false, 10000000, false);
         CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID())
    )
    {
        writer.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory, txn));
        while (ci.hasNext())
        {
            writer.append(ci.next());
            if (!checked && writer.currentWriter().getFilePointer() > 15000000)
            {
                checked = true;
                ColumnFamilyStore.ViewFragment viewFragment = cfs.select(View.selectFunction(SSTableSet.CANONICAL));
                // canonical view should have only one SSTable which is not opened early.
                assertEquals(1, viewFragment.sstables.size());
                SSTableReader sstable = viewFragment.sstables.get(0);
                assertEquals(s.descriptor, sstable.descriptor);
                assertTrue("Found early opened SSTable in canonical view: " + sstable.getFilename(), sstable.openReason != SSTableReader.OpenReason.EARLY);
            }
        }
    }
    truncateCF();
    validateCFS(cfs);
}
 

/**
 * emulates anticompaction - writing from one source sstable to two new sstables
 *
 * @throws IOException
 */
@Test
public void testTwoWriters() throws IOException
{
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    truncate(cfs);

    SSTableReader s = writeFile(cfs, 1000);
    cfs.addSSTable(s);
    Set<SSTableReader> sstables = Sets.newHashSet(s);
    assertEquals(1, sstables.size());
    int nowInSec = FBUtilities.nowInSeconds();
    try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(sstables);
         LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN);
         SSTableRewriter writer = new SSTableRewriter(txn, 1000, false, false);
         SSTableRewriter writer2 = new SSTableRewriter(txn, 1000, false, false);
         CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(nowInSec));
         CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID())
         )
    {
        writer.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory, txn));
        writer2.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory, txn));
        while (ci.hasNext())
        {
            if (writer.currentWriter().getFilePointer() < 15000000)
                writer.append(ci.next());
            else
                writer2.append(ci.next());
        }
        for (int i = 0; i < 5000; i++)
            assertFalse(Util.getOnlyPartition(Util.cmd(cfs, ByteBufferUtil.bytes(i)).build()).isEmpty());
    }
    truncateCF();
    validateCFS(cfs);
}
 

@Test
public void getPositionsTest() throws InterruptedException
{
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    truncate(cfs);

    SSTableReader s = writeFile(cfs, 1000);
    cfs.addSSTable(s);
    Set<SSTableReader> sstables = new HashSet<>(cfs.getLiveSSTables());
    assertEquals(1, sstables.size());

    int nowInSec = FBUtilities.nowInSeconds();
    boolean checked = false;
    try (AbstractCompactionStrategy.ScannerList scanners = cfs.getCompactionStrategyManager().getScanners(sstables);
         LifecycleTransaction txn = cfs.getTracker().tryModify(sstables, OperationType.UNKNOWN);
         SSTableRewriter writer = new SSTableRewriter(txn, 1000, false, 10000000, false);
         CompactionController controller = new CompactionController(cfs, sstables, cfs.gcBefore(nowInSec));
         CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, scanners.scanners, controller, nowInSec, UUIDGen.getTimeUUID()))
    {
        writer.switchWriter(getWriter(cfs, sstables.iterator().next().descriptor.directory, txn));
        while (ci.hasNext())
        {
            UnfilteredRowIterator row = ci.next();
            writer.append(row);
            if (!checked && writer.currentWriter().getFilePointer() > 1500000)
            {
                checked = true;
                for (SSTableReader sstable : cfs.getLiveSSTables())
                {
                    if (sstable.openReason == SSTableReader.OpenReason.EARLY)
                    {
                        SSTableReader c = txn.current(sstables.iterator().next());
                        Collection<Range<Token>> r = Arrays.asList(new Range<>(cfs.getPartitioner().getMinimumToken(), cfs.getPartitioner().getMinimumToken()));
                        List<Pair<Long, Long>> tmplinkPositions = sstable.getPositionsForRanges(r);
                        List<Pair<Long, Long>> compactingPositions = c.getPositionsForRanges(r);
                        assertEquals(1, tmplinkPositions.size());
                        assertEquals(1, compactingPositions.size());
                        assertEquals(0, tmplinkPositions.get(0).left.longValue());
                        // make sure we have no overlap between the early opened file and the compacting one:
                        assertEquals(tmplinkPositions.get(0).right.longValue(), compactingPositions.get(0).left.longValue());
                        assertEquals(c.uncompressedLength(), compactingPositions.get(0).right.longValue());
                    }
                }
            }
        }
        assertTrue(checked);
        writer.finish();
    }
    LifecycleTransaction.waitForDeletions();
    validateCFS(cfs);
    int filecounts = assertFileCounts(sstables.iterator().next().descriptor.directory.list());
    assertEquals(1, filecounts);
    truncate(cfs);
}
 

@Test
public void testNumberOfFilesAndSizes() throws Exception
{
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    truncate(cfs);

    SSTableReader s = writeFile(cfs, 1000);
    cfs.addSSTable(s);
    long startStorageMetricsLoad = StorageMetrics.load.getCount();
    long sBytesOnDisk = s.bytesOnDisk();
    Set<SSTableReader> compacting = Sets.newHashSet(s);

    List<SSTableReader> sstables;
    int files = 1;
    try (ISSTableScanner scanner = s.getScanner();
         CompactionController controller = new CompactionController(cfs, compacting, 0);
         LifecycleTransaction txn = cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN);
         SSTableRewriter rewriter = new SSTableRewriter(txn, 1000, false, 10000000, false);
         CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID()))
    {
        rewriter.switchWriter(getWriter(cfs, s.descriptor.directory, txn));

        while(ci.hasNext())
        {
            rewriter.append(ci.next());
            if (rewriter.currentWriter().getOnDiskFilePointer() > 25000000)
            {
                rewriter.switchWriter(getWriter(cfs, s.descriptor.directory, txn));
                files++;
                assertEquals(cfs.getLiveSSTables().size(), files); // we have one original file plus the ones we have switched out.
                assertEquals(s.bytesOnDisk(), cfs.metric.liveDiskSpaceUsed.getCount());
                assertEquals(s.bytesOnDisk(), cfs.metric.totalDiskSpaceUsed.getCount());

            }
        }
        sstables = rewriter.finish();
    }

    LifecycleTransaction.waitForDeletions();

    long sum = 0;
    for (SSTableReader x : cfs.getLiveSSTables())
        sum += x.bytesOnDisk();
    assertEquals(sum, cfs.metric.liveDiskSpaceUsed.getCount());
    assertEquals(startStorageMetricsLoad - sBytesOnDisk + sum, StorageMetrics.load.getCount());
    assertEquals(files, sstables.size());
    assertEquals(files, cfs.getLiveSSTables().size());
    LifecycleTransaction.waitForDeletions();

    // tmplink and tmp files should be gone:
    assertEquals(sum, cfs.metric.totalDiskSpaceUsed.getCount());
    assertFileCounts(s.descriptor.directory.list());
    validateCFS(cfs);
}
 

@Test
public void testNumberOfFiles_dont_clean_readers() throws Exception
{
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    truncate(cfs);

    SSTableReader s = writeFile(cfs, 1000);
    cfs.addSSTable(s);

    Set<SSTableReader> compacting = Sets.newHashSet(s);

    List<SSTableReader> sstables;
    int files = 1;
    try (ISSTableScanner scanner = s.getScanner();
         CompactionController controller = new CompactionController(cfs, compacting, 0);
         LifecycleTransaction txn = cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN);
         SSTableRewriter rewriter = new SSTableRewriter(txn, 1000, false, 10000000, false);
         CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID()))
    {
        rewriter.switchWriter(getWriter(cfs, s.descriptor.directory, txn));

        while(ci.hasNext())
        {
            rewriter.append(ci.next());
            if (rewriter.currentWriter().getOnDiskFilePointer() > 25000000)
            {
                rewriter.switchWriter(getWriter(cfs, s.descriptor.directory, txn));
                files++;
                assertEquals(cfs.getLiveSSTables().size(), files); // we have one original file plus the ones we have switched out.
            }
        }
        sstables = rewriter.finish();
    }

    assertEquals(files, sstables.size());
    assertEquals(files, cfs.getLiveSSTables().size());
    LifecycleTransaction.waitForDeletions();

    assertFileCounts(s.descriptor.directory.list());
    validateCFS(cfs);
}
 

@Test
public void testNumberOfFiles_finish_empty_new_writer() throws Exception
{
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    truncate(cfs);

    SSTableReader s = writeFile(cfs, 1000);
    cfs.addSSTable(s);

    Set<SSTableReader> compacting = Sets.newHashSet(s);

    int files = 1;
    try (ISSTableScanner scanner = s.getScanner();
         CompactionController controller = new CompactionController(cfs, compacting, 0);
         LifecycleTransaction txn = cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN);
         SSTableRewriter rewriter = new SSTableRewriter(txn, 1000, false, 10000000, false);
         CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID()))
    {
        rewriter.switchWriter(getWriter(cfs, s.descriptor.directory, txn));
        while(ci.hasNext())
        {
            rewriter.append(ci.next());
            if (rewriter.currentWriter().getFilePointer() > 2500000)
            {
                rewriter.switchWriter(getWriter(cfs, s.descriptor.directory, txn));
                files++;
                assertEquals(cfs.getLiveSSTables().size(), files); // we have one original file plus the ones we have switched out.
            }
            if (files == 3)
            {
                //testing to finish when we have nothing written in the new file
                rewriter.finish();
                break;
            }
        }
    }

    LifecycleTransaction.waitForDeletions();

    assertEquals(files - 1, cfs.getLiveSSTables().size()); // we never wrote anything to the last file
    assertFileCounts(s.descriptor.directory.list());
    validateCFS(cfs);
}
 

@Test
public void testSmallFiles() throws Exception
{
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    truncate(cfs);
    cfs.disableAutoCompaction();

    SSTableReader s = writeFile(cfs, 400);
    cfs.addSSTable(s);
    Set<SSTableReader> compacting = Sets.newHashSet(s);

    List<SSTableReader> sstables;
    int files = 1;
    try (ISSTableScanner scanner = s.getScanner();
         CompactionController controller = new CompactionController(cfs, compacting, 0);
         LifecycleTransaction txn = cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN);
         SSTableRewriter rewriter = new SSTableRewriter(txn, 1000, false, 1000000, false);
         CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID()))
    {
        rewriter.switchWriter(getWriter(cfs, s.descriptor.directory, txn));
        while(ci.hasNext())
        {
            rewriter.append(ci.next());
            if (rewriter.currentWriter().getOnDiskFilePointer() > 2500000)
            {
                assertEquals(files, cfs.getLiveSSTables().size()); // all files are now opened early
                rewriter.switchWriter(getWriter(cfs, s.descriptor.directory, txn));
                files++;
            }
        }

        sstables = rewriter.finish();
    }
    assertEquals(files, sstables.size());
    assertEquals(files, cfs.getLiveSSTables().size());
    LifecycleTransaction.waitForDeletions();
    assertFileCounts(s.descriptor.directory.list());

    validateCFS(cfs);
}
 

private void testAbortHelper(boolean earlyException, boolean offline) throws Exception
{
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    truncate(cfs);
    SSTableReader s = writeFile(cfs, 1000);
    if (!offline)
        cfs.addSSTable(s);
    Set<SSTableReader> compacting = Sets.newHashSet(s);
    try (ISSTableScanner scanner = compacting.iterator().next().getScanner();
         CompactionController controller = new CompactionController(cfs, compacting, 0);
         LifecycleTransaction txn = offline ? LifecycleTransaction.offline(OperationType.UNKNOWN, compacting)
                                   : cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN);
         SSTableRewriter rewriter = new SSTableRewriter(txn, 1000, offline, 10000000, false);
         CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID())
    )
    {
        rewriter.switchWriter(getWriter(cfs, s.descriptor.directory, txn));
        while (ci.hasNext())
        {
            rewriter.append(ci.next());
            if (rewriter.currentWriter().getOnDiskFilePointer() > 25000000)
            {
                rewriter.switchWriter(getWriter(cfs, s.descriptor.directory, txn));
            }
        }
        try
        {
            rewriter.throwDuringPrepare(earlyException);
            rewriter.prepareToCommit();
        }
        catch (Throwable t)
        {
            rewriter.abort();
        }
    }
    finally
    {
        if (offline)
            s.selfRef().release();
    }

    LifecycleTransaction.waitForDeletions();

    int filecount = assertFileCounts(s.descriptor.directory.list());
    assertEquals(filecount, 1);
    if (!offline)
    {
        assertEquals(1, cfs.getLiveSSTables().size());
        validateCFS(cfs);
        truncate(cfs);
    }
    else
    {
        assertEquals(0, cfs.getLiveSSTables().size());
        cfs.truncateBlocking();
    }
    filecount = assertFileCounts(s.descriptor.directory.list());
    if (offline)
    {
        // the file is not added to the CFS, therefore not truncated away above
        assertEquals(1, filecount);
        for (File f : s.descriptor.directory.listFiles())
        {
            FileUtils.deleteRecursive(f);
        }
        filecount = assertFileCounts(s.descriptor.directory.list());
    }

    assertEquals(0, filecount);
    truncate(cfs);
}
 

@Test
public void testAllKeysReadable() throws Exception
{
    Keyspace keyspace = Keyspace.open(KEYSPACE);
    ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF);
    truncate(cfs);
    for (int i = 0; i < 100; i++)
    {
        String key = Integer.toString(i);

        for (int j = 0; j < 10; j++)
            new RowUpdateBuilder(cfs.metadata, 100, key)
                .clustering(Integer.toString(j))
                .add("val", ByteBufferUtil.EMPTY_BYTE_BUFFER)
                .build()
                .apply();
    }
    cfs.forceBlockingFlush();
    cfs.forceMajorCompaction();
    validateKeys(keyspace);

    assertEquals(1, cfs.getLiveSSTables().size());
    SSTableReader s = cfs.getLiveSSTables().iterator().next();
    Set<SSTableReader> compacting = new HashSet<>();
    compacting.add(s);

    int keyCount = 0;
    try (ISSTableScanner scanner = compacting.iterator().next().getScanner();
         CompactionController controller = new CompactionController(cfs, compacting, 0);
         LifecycleTransaction txn = cfs.getTracker().tryModify(compacting, OperationType.UNKNOWN);
         SSTableRewriter rewriter = new SSTableRewriter(txn, 1000, false, 1, false);
         CompactionIterator ci = new CompactionIterator(OperationType.COMPACTION, Collections.singletonList(scanner), controller, FBUtilities.nowInSeconds(), UUIDGen.getTimeUUID())
    )
    {
        rewriter.switchWriter(getWriter(cfs, s.descriptor.directory, txn));
        while (ci.hasNext())
        {
            rewriter.append(ci.next());
            if (keyCount % 10 == 0)
            {
                rewriter.switchWriter(getWriter(cfs, s.descriptor.directory, txn));
            }
            keyCount++;
            validateKeys(keyspace);
        }
        rewriter.finish();
    }
    validateKeys(keyspace);
    LifecycleTransaction.waitForDeletions();
    validateCFS(cfs);
    truncate(cfs);
}