Java Unit类代码示例

import io.airlift.units.DataSize.Unit; //导入依赖的package包/类
public static OrcRecordReader createReader(OrcDataSource dataSource, List<Long> columnIds, List<Type> types)
        throws IOException
{
    OrcReader orcReader = new OrcReader(dataSource, new OrcMetadataReader(), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE));

    List<String> columnNames = orcReader.getColumnNames();
    assertEquals(columnNames.size(), columnIds.size());

    Map<Integer, Type> includedColumns = new HashMap<>();
    int ordinal = 0;
    for (long columnId : columnIds) {
        assertEquals(columnNames.get(ordinal), String.valueOf(columnId));
        includedColumns.put(ordinal, types.get(ordinal));
        ordinal++;
    }

    return createRecordReader(orcReader, includedColumns);
}
 

import io.airlift.units.DataSize.Unit; //导入依赖的package包/类
@Test
public void testIntegration()
        throws IOException
{
    // tiny file
    TestingOrcDataSource orcDataSource = new TestingOrcDataSource(
            new FileOrcDataSource(tempFile.getFile(), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE)));
    doIntegration(orcDataSource, new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE));
    assertEquals(orcDataSource.getReadCount(), 1); // read entire file at once

    // tiny stripes
    orcDataSource = new TestingOrcDataSource(
            new FileOrcDataSource(tempFile.getFile(), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE)));
    doIntegration(orcDataSource, new DataSize(400, Unit.KILOBYTE), new DataSize(400, Unit.KILOBYTE));
    assertEquals(orcDataSource.getReadCount(), 3); // footer, first few stripes, last few stripes
}
 

import io.airlift.units.DataSize.Unit; //导入依赖的package包/类
public static OrcRecordReader createReaderNoRows(OrcDataSource dataSource)
        throws IOException
{
    OrcReader orcReader = new OrcReader(dataSource, new OrcMetadataReader(), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE));

    assertEquals(orcReader.getColumnNames().size(), 0);

    return createRecordReader(orcReader, ImmutableMap.of());
}
 

import io.airlift.units.DataSize.Unit; //导入依赖的package包/类
@Override
public OperatorFactory duplicate()
{
    return new HashAggregationOperatorFactory(
            operatorId,
            planNodeId,
            groupByTypes,
            groupByChannels,
            step,
            accumulatorFactories,
            maskChannel,
            hashChannel,
            expectedGroups,
            new DataSize(maxPartialMemory, Unit.BYTE));
}
 

import io.airlift.units.DataSize.Unit; //导入依赖的package包/类
@Test
public void testHappyPath()
        throws Exception
{
    DataSize maxResponseSize = new DataSize(10, Unit.MEGABYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);

    URI location = URI.create("http://localhost:8080");
    processor.addPage(location, createPage(1));
    processor.addPage(location, createPage(2));
    processor.addPage(location, createPage(3));
    processor.setComplete(location);

    @SuppressWarnings("resource")
    ExchangeClient exchangeClient = new ExchangeClient(blockEncodingManager, new DataSize(32, Unit.MEGABYTE), maxResponseSize, 1, new Duration(1, TimeUnit.MINUTES), new TestingHttpClient(processor, executor), executor, deltaMemoryInBytes -> { });

    exchangeClient.addLocation(location);
    exchangeClient.noMoreLocations();

    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(1));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(2));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(3));
    assertNull(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)));
    assertEquals(exchangeClient.isClosed(), true);
    if (exchangeClient.getStatus().getBufferedPages() != 0) {
        assertEquals(exchangeClient.getStatus().getBufferedPages(), 0);
    }
    assertTrue(exchangeClient.getStatus().getBufferedBytes() == 0);

    // client should have sent only 2 requests: one to get all pages and once to get the done signal
    assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "closed", 3, 3, 3, "not scheduled");
}
 

import io.airlift.units.DataSize.Unit; //导入依赖的package包/类
@Test
public void testClose()
        throws Exception
{
    DataSize maxResponseSize = new DataSize(1, Unit.BYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);

    URI location = URI.create("http://localhost:8080");
    processor.addPage(location, createPage(1));
    processor.addPage(location, createPage(2));
    processor.addPage(location, createPage(3));

    @SuppressWarnings("resource")
    ExchangeClient exchangeClient = new ExchangeClient(blockEncodingManager, new DataSize(1, Unit.BYTE), maxResponseSize, 1, new Duration(1, TimeUnit.MINUTES), new TestingHttpClient(processor, newCachedThreadPool(daemonThreadsNamed("test-%s"))), executor, deltaMemoryInBytes -> { });
    exchangeClient.addLocation(location);
    exchangeClient.noMoreLocations();

    // fetch a page
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(1));

    // close client while pages are still available
    exchangeClient.close();
    assertEquals(exchangeClient.isClosed(), true);
    assertNull(exchangeClient.getNextPage(new Duration(0, TimeUnit.SECONDS)));
    assertEquals(exchangeClient.getStatus().getBufferedPages(), 0);
    assertEquals(exchangeClient.getStatus().getBufferedBytes(), 0);

    // client should have sent only 2 requests: one to get all pages and once to get the done signal
    PageBufferClientStatus clientStatus = exchangeClient.getStatus().getPageBufferClientStatuses().get(0);
    assertEquals(clientStatus.getUri(), location);
    assertEquals(clientStatus.getState(), "closed", "status");
    assertEquals(clientStatus.getHttpRequestState(), "not scheduled", "httpRequestState");
}
 

import io.airlift.units.DataSize.Unit; //导入依赖的package包/类
@Test
public void testLifecycle()
        throws Exception
{
    CyclicBarrier beforeRequest = new CyclicBarrier(2);
    CyclicBarrier afterRequest = new CyclicBarrier(2);
    StaticRequestProcessor processor = new StaticRequestProcessor(beforeRequest, afterRequest);
    processor.setResponse(new TestingResponse(HttpStatus.NO_CONTENT, ImmutableListMultimap.<String, String>of(), new byte[0]));

    CyclicBarrier requestComplete = new CyclicBarrier(2);
    TestingClientCallback callback = new TestingClientCallback(requestComplete);

    URI location = URI.create("http://localhost:8080");
    HttpPageBufferClient client = new HttpPageBufferClient(new TestingHttpClient(processor, executor),
            new DataSize(10, Unit.MEGABYTE),
            new Duration(1, TimeUnit.MINUTES),
            location,
            callback,
            blockEncodingManager,
            executor,
            Stopwatch.createUnstarted());

    assertStatus(client, location, "queued", 0, 0, 0, 0, "not scheduled");

    client.scheduleRequest();
    beforeRequest.await(10, TimeUnit.SECONDS);
    assertStatus(client, location, "running", 0, 1, 0, 0, "PROCESSING_REQUEST");
    assertEquals(client.isRunning(), true);
    afterRequest.await(10, TimeUnit.SECONDS);

    requestComplete.await(10, TimeUnit.SECONDS);
    assertStatus(client, location, "queued", 0, 1, 1, 1, "not scheduled");

    client.close();
    assertStatus(client, location, "closed", 0, 1, 1, 1, "not scheduled");
}
 

import io.airlift.units.DataSize.Unit; //导入依赖的package包/类
@Test
public void testCloseDuringPendingRequest()
        throws Exception
{
    CyclicBarrier beforeRequest = new CyclicBarrier(2);
    CyclicBarrier afterRequest = new CyclicBarrier(2);
    StaticRequestProcessor processor = new StaticRequestProcessor(beforeRequest, afterRequest);
    processor.setResponse(new TestingResponse(HttpStatus.NO_CONTENT, ImmutableListMultimap.<String, String>of(), new byte[0]));

    CyclicBarrier requestComplete = new CyclicBarrier(2);
    TestingClientCallback callback = new TestingClientCallback(requestComplete);

    URI location = URI.create("http://localhost:8080");
    HttpPageBufferClient client = new HttpPageBufferClient(new TestingHttpClient(processor, executor),
            new DataSize(10, Unit.MEGABYTE),
            new Duration(1, TimeUnit.MINUTES),
            location,
            callback,
            blockEncodingManager,
            executor,
            Stopwatch.createUnstarted());

    assertStatus(client, location, "queued", 0, 0, 0, 0, "not scheduled");

    // send request
    client.scheduleRequest();
    beforeRequest.await(10, TimeUnit.SECONDS);
    assertStatus(client, location, "running", 0, 1, 0, 0, "PROCESSING_REQUEST");
    assertEquals(client.isRunning(), true);
    // request is pending, now close it
    client.close();

    try {
        requestComplete.await(10, TimeUnit.SECONDS);
    }
    catch (BrokenBarrierException ignored) {
    }
    assertStatus(client, location, "closed", 0, 1, 1, 1, "not scheduled");
}
 

import io.airlift.units.DataSize.Unit; //导入依赖的package包/类
@Test(expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded local memory limit of 10B")
public void testMemoryLimit()
        throws Exception
{
    List<Page> input = rowPagesBuilder(BIGINT, DOUBLE)
            .row(1, 0.1)
            .row(2, 0.2)
            .pageBreak()
            .row(-1, -0.1)
            .row(4, 0.4)
            .build();

    DriverContext driverContext = createTaskContext(executor, TEST_SESSION, new DataSize(10, Unit.BYTE))
            .addPipelineContext(true, true)
            .addDriverContext();

    WindowOperatorFactory operatorFactory = createFactoryUnbounded(
            ImmutableList.of(BIGINT, DOUBLE),
            Ints.asList(1),
            ROW_NUMBER,
            Ints.asList(),
            Ints.asList(0),
            ImmutableList.copyOf(new SortOrder[] {SortOrder.ASC_NULLS_LAST}));

    Operator operator = operatorFactory.createOperator(driverContext);

    toPages(operator, input);
}
 

import io.airlift.units.DataSize.Unit; //导入依赖的package包/类
@Test(expectedExceptions = ExceededMemoryLimitException.class, expectedExceptionsMessageRegExp = "Query exceeded local memory limit of 10B")
public void testMemoryLimit()
        throws Exception
{
    List<Page> input = rowPagesBuilder(BIGINT, DOUBLE)
            .row(1, 0.1)
            .row(2, 0.2)
            .pageBreak()
            .row(-1, -0.1)
            .row(4, 0.4)
            .build();

    DriverContext driverContext = createTaskContext(executor, TEST_SESSION, new DataSize(10, Unit.BYTE))
            .addPipelineContext(true, true)
            .addDriverContext();

    OrderByOperatorFactory operatorFactory = new OrderByOperatorFactory(
            0,
            new PlanNodeId("test"),
            ImmutableList.of(BIGINT, DOUBLE),
            ImmutableList.of(1),
            10,
            ImmutableList.of(0),
            ImmutableList.of(ASC_NULLS_LAST));

    Operator operator = operatorFactory.createOperator(driverContext);

    toPages(operator, input);
}
 

import io.airlift.units.DataSize.Unit; //导入依赖的package包/类
@Test
public void testSimpleQuery()
        throws Exception
{
    try (SqlTaskManager sqlTaskManager = createSqlTaskManager(new TaskManagerConfig())) {
        TaskId taskId = TASK_ID;
        TaskInfo taskInfo = sqlTaskManager.updateTask(TEST_SESSION,
                taskId,
                PLAN_FRAGMENT,
                ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(SPLIT), true)),
                INITIAL_EMPTY_OUTPUT_BUFFERS.withBuffer(OUT, 0).withNoMoreBufferIds());
        assertEquals(taskInfo.getState(), TaskState.RUNNING);

        taskInfo = sqlTaskManager.getTaskInfo(taskId);
        assertEquals(taskInfo.getState(), TaskState.RUNNING);

        BufferResult results = sqlTaskManager.getTaskResults(taskId, OUT, 0, new DataSize(1, Unit.MEGABYTE)).get();
        assertEquals(results.isBufferComplete(), false);
        assertEquals(results.getPages().size(), 1);
        assertEquals(results.getPages().get(0).getPositionCount(), 1);

        results = sqlTaskManager.getTaskResults(taskId, OUT, results.getToken() + results.getPages().size(), new DataSize(1, Unit.MEGABYTE)).get();
        assertEquals(results.isBufferComplete(), true);
        assertEquals(results.getPages().size(), 0);

        // complete the task by calling abort on it
        TaskInfo info = sqlTaskManager.abortTaskResults(taskId, OUT);
        assertEquals(info.getOutputBuffers().getState(), SharedBuffer.BufferState.FINISHED);

        taskInfo = sqlTaskManager.getTaskInfo(taskId, taskInfo.getState()).get(1, TimeUnit.SECONDS);
        assertEquals(taskInfo.getState(), TaskState.FINISHED);
        taskInfo = sqlTaskManager.getTaskInfo(taskId);
        assertEquals(taskInfo.getState(), TaskState.FINISHED);
    }
}
 

import io.airlift.units.DataSize.Unit; //导入依赖的package包/类
public MockRemoteTask(TaskId taskId, PlanFragment fragment, Executor executor) {
    this.taskStateMachine = new TaskStateMachine(checkNotNull(taskId, "taskId is null"), checkNotNull(executor,
            "executor is null"));

    Session session = new Session("user", "source", "catalog", "schema", "address", "agent");
    this.taskContext = new TaskContext(taskStateMachine, executor, session, new DataSize(256, MEGABYTE),
            new DataSize(1, MEGABYTE), true);

    this.location = URI.create("fake://task/" + taskId);

    this.sharedBuffer = new SharedBuffer(checkNotNull(new DataSize(1, Unit.BYTE), "maxBufferSize is null"),
            INITIAL_EMPTY_OUTPUT_BUFFERS);
    this.fragment = checkNotNull(fragment, "fragment is null");
}
 

import io.airlift.units.DataSize.Unit; //导入依赖的package包/类
@Test
public void testWrapWithCacheIfTinyStripes()
        throws IOException
{
    DataSize maxMergeDistance = new DataSize(1, Unit.MEGABYTE);
    DataSize maxReadSize = new DataSize(8, Unit.MEGABYTE);

    OrcDataSource actual = wrapWithCacheIfTinyStripes(
            FakeOrcDataSource.INSTANCE,
            ImmutableList.of(),
            maxMergeDistance,
            maxReadSize);
    assertInstanceOf(actual, CachingOrcDataSource.class);

    actual = wrapWithCacheIfTinyStripes(
            FakeOrcDataSource.INSTANCE,
            ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10)),
            maxMergeDistance,
            maxReadSize);
    assertInstanceOf(actual, CachingOrcDataSource.class);

    actual = wrapWithCacheIfTinyStripes(
            FakeOrcDataSource.INSTANCE,
            ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10), new StripeInformation(123, 33, 10, 10, 10), new StripeInformation(123, 63, 10, 10, 10)),
            maxMergeDistance,
            maxReadSize);
    assertInstanceOf(actual, CachingOrcDataSource.class);

    actual = wrapWithCacheIfTinyStripes(
            FakeOrcDataSource.INSTANCE,
            ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10), new StripeInformation(123, 33, 10, 10, 10), new StripeInformation(123, 63, 1048576 * 8 - 20, 10, 10)),
            maxMergeDistance,
            maxReadSize);
    assertInstanceOf(actual, CachingOrcDataSource.class);

    actual = wrapWithCacheIfTinyStripes(
            FakeOrcDataSource.INSTANCE,
            ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10), new StripeInformation(123, 33, 10, 10, 10), new StripeInformation(123, 63, 1048576 * 8 - 20 + 1, 10, 10)),
            maxMergeDistance,
            maxReadSize);
    assertNotInstanceOf(actual, CachingOrcDataSource.class);
}
 

import io.airlift.units.DataSize.Unit; //导入依赖的package包/类
@Test
public void testTinyStripesReadCacheAt()
        throws IOException
{
    DataSize maxMergeDistance = new DataSize(1, Unit.MEGABYTE);
    DataSize maxReadSize = new DataSize(8, Unit.MEGABYTE);

    TestingOrcDataSource testingOrcDataSource = new TestingOrcDataSource(FakeOrcDataSource.INSTANCE);
    CachingOrcDataSource cachingOrcDataSource = new CachingOrcDataSource(
            testingOrcDataSource,
            createTinyStripesRangeFinder(
                    ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10), new StripeInformation(123, 33, 10, 10, 10), new StripeInformation(123, 63, 1048576 * 8 - 20, 10, 10)),
                    maxMergeDistance,
                    maxReadSize));
    cachingOrcDataSource.readCacheAt(3);
    assertEquals(testingOrcDataSource.getLastReadRanges(), ImmutableList.of(new DiskRange(3, 60)));
    cachingOrcDataSource.readCacheAt(63);
    assertEquals(testingOrcDataSource.getLastReadRanges(), ImmutableList.of(new DiskRange(63, 8 * 1048576)));

    testingOrcDataSource = new TestingOrcDataSource(FakeOrcDataSource.INSTANCE);
    cachingOrcDataSource = new CachingOrcDataSource(
            testingOrcDataSource,
            createTinyStripesRangeFinder(
                    ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10), new StripeInformation(123, 33, 10, 10, 10), new StripeInformation(123, 63, 1048576 * 8 - 20, 10, 10)),
                    maxMergeDistance,
                    maxReadSize));
    cachingOrcDataSource.readCacheAt(62); // read at the end of a stripe
    assertEquals(testingOrcDataSource.getLastReadRanges(), ImmutableList.of(new DiskRange(3, 60)));
    cachingOrcDataSource.readCacheAt(63);
    assertEquals(testingOrcDataSource.getLastReadRanges(), ImmutableList.of(new DiskRange(63, 8 * 1048576)));

    testingOrcDataSource = new TestingOrcDataSource(FakeOrcDataSource.INSTANCE);
    cachingOrcDataSource = new CachingOrcDataSource(
            testingOrcDataSource,
            createTinyStripesRangeFinder(
                    ImmutableList.of(new StripeInformation(123, 3, 1, 0, 0), new StripeInformation(123, 4, 1048576, 1048576, 1048576 * 3), new StripeInformation(123, 4 + 1048576 * 5, 1048576, 1048576, 1048576)),
                    maxMergeDistance,
                    maxReadSize));
    cachingOrcDataSource.readCacheAt(3);
    assertEquals(testingOrcDataSource.getLastReadRanges(), ImmutableList.of(new DiskRange(3, 1 + 1048576 * 5)));
    cachingOrcDataSource.readCacheAt(4 + 1048576 * 5);
    assertEquals(testingOrcDataSource.getLastReadRanges(), ImmutableList.of(new DiskRange(4 + 1048576 * 5, 3 * 1048576)));
}
 

import io.airlift.units.DataSize.Unit; //导入依赖的package包/类
@Test(timeOut = 10000)
public void testAddLocation()
        throws Exception
{
    DataSize maxResponseSize = new DataSize(10, Unit.MEGABYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);

    @SuppressWarnings("resource")
    ExchangeClient exchangeClient = new ExchangeClient(blockEncodingManager, new DataSize(32, Unit.MEGABYTE), maxResponseSize, 1, new Duration(1, TimeUnit.MINUTES), new TestingHttpClient(processor, newCachedThreadPool(daemonThreadsNamed("test-%s"))), executor, deltaMemoryInBytes -> { });

    URI location1 = URI.create("http://localhost:8081/foo");
    processor.addPage(location1, createPage(1));
    processor.addPage(location1, createPage(2));
    processor.addPage(location1, createPage(3));
    processor.setComplete(location1);
    exchangeClient.addLocation(location1);

    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(1));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(2));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(3));

    assertNull(exchangeClient.getNextPage(new Duration(10, TimeUnit.MILLISECONDS)));
    assertEquals(exchangeClient.isClosed(), false);

    URI location2 = URI.create("http://localhost:8082/bar");
    processor.addPage(location2, createPage(4));
    processor.addPage(location2, createPage(5));
    processor.addPage(location2, createPage(6));
    processor.setComplete(location2);
    exchangeClient.addLocation(location2);

    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(4));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(5));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(6));

    assertNull(exchangeClient.getNextPage(new Duration(10, TimeUnit.MILLISECONDS)));
    assertEquals(exchangeClient.isClosed(), false);

    exchangeClient.noMoreLocations();
    // The transition to closed may happen asynchronously, since it requires that all the HTTP clients
    // receive a final GONE response, so just spin until it's closed or the test times out.
    while (!exchangeClient.isClosed()) {
        Thread.sleep(1);
    }

    ImmutableMap<URI, PageBufferClientStatus> statuses = uniqueIndex(exchangeClient.getStatus().getPageBufferClientStatuses(), PageBufferClientStatus::getUri);
    assertStatus(statuses.get(location1), location1, "closed", 3, 3, 3, "not scheduled");
    assertStatus(statuses.get(location2), location2, "closed", 3, 3, 3, "not scheduled");
}