Java Unit.MEGABYTE属性代码示例

public static OrcRecordReader createReader(OrcDataSource dataSource, List<Long> columnIds, List<Type> types)
        throws IOException
{
    OrcReader orcReader = new OrcReader(dataSource, new OrcMetadataReader(), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE));

    List<String> columnNames = orcReader.getColumnNames();
    assertEquals(columnNames.size(), columnIds.size());

    Map<Integer, Type> includedColumns = new HashMap<>();
    int ordinal = 0;
    for (long columnId : columnIds) {
        assertEquals(columnNames.get(ordinal), String.valueOf(columnId));
        includedColumns.put(ordinal, types.get(ordinal));
        ordinal++;
    }

    return createRecordReader(orcReader, includedColumns);
}
 

@Test
public void testIntegration()
        throws IOException
{
    // tiny file
    TestingOrcDataSource orcDataSource = new TestingOrcDataSource(
            new FileOrcDataSource(tempFile.getFile(), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE)));
    doIntegration(orcDataSource, new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE));
    assertEquals(orcDataSource.getReadCount(), 1); // read entire file at once

    // tiny stripes
    orcDataSource = new TestingOrcDataSource(
            new FileOrcDataSource(tempFile.getFile(), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE)));
    doIntegration(orcDataSource, new DataSize(400, Unit.KILOBYTE), new DataSize(400, Unit.KILOBYTE));
    assertEquals(orcDataSource.getReadCount(), 3); // footer, first few stripes, last few stripes
}
 

public static OrcRecordReader createReaderNoRows(OrcDataSource dataSource)
        throws IOException
{
    OrcReader orcReader = new OrcReader(dataSource, new OrcMetadataReader(), new DataSize(1, Unit.MEGABYTE), new DataSize(1, Unit.MEGABYTE));

    assertEquals(orcReader.getColumnNames().size(), 0);

    return createRecordReader(orcReader, ImmutableMap.of());
}
 

@Test
public void testHappyPath()
        throws Exception
{
    DataSize maxResponseSize = new DataSize(10, Unit.MEGABYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);

    URI location = URI.create("http://localhost:8080");
    processor.addPage(location, createPage(1));
    processor.addPage(location, createPage(2));
    processor.addPage(location, createPage(3));
    processor.setComplete(location);

    @SuppressWarnings("resource")
    ExchangeClient exchangeClient = new ExchangeClient(blockEncodingManager, new DataSize(32, Unit.MEGABYTE), maxResponseSize, 1, new Duration(1, TimeUnit.MINUTES), new TestingHttpClient(processor, executor), executor, deltaMemoryInBytes -> { });

    exchangeClient.addLocation(location);
    exchangeClient.noMoreLocations();

    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(1));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(2));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(3));
    assertNull(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)));
    assertEquals(exchangeClient.isClosed(), true);
    if (exchangeClient.getStatus().getBufferedPages() != 0) {
        assertEquals(exchangeClient.getStatus().getBufferedPages(), 0);
    }
    assertTrue(exchangeClient.getStatus().getBufferedBytes() == 0);

    // client should have sent only 2 requests: one to get all pages and once to get the done signal
    assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "closed", 3, 3, 3, "not scheduled");
}
 

@Test
public void testLifecycle()
        throws Exception
{
    CyclicBarrier beforeRequest = new CyclicBarrier(2);
    CyclicBarrier afterRequest = new CyclicBarrier(2);
    StaticRequestProcessor processor = new StaticRequestProcessor(beforeRequest, afterRequest);
    processor.setResponse(new TestingResponse(HttpStatus.NO_CONTENT, ImmutableListMultimap.<String, String>of(), new byte[0]));

    CyclicBarrier requestComplete = new CyclicBarrier(2);
    TestingClientCallback callback = new TestingClientCallback(requestComplete);

    URI location = URI.create("http://localhost:8080");
    HttpPageBufferClient client = new HttpPageBufferClient(new TestingHttpClient(processor, executor),
            new DataSize(10, Unit.MEGABYTE),
            new Duration(1, TimeUnit.MINUTES),
            location,
            callback,
            blockEncodingManager,
            executor,
            Stopwatch.createUnstarted());

    assertStatus(client, location, "queued", 0, 0, 0, 0, "not scheduled");

    client.scheduleRequest();
    beforeRequest.await(10, TimeUnit.SECONDS);
    assertStatus(client, location, "running", 0, 1, 0, 0, "PROCESSING_REQUEST");
    assertEquals(client.isRunning(), true);
    afterRequest.await(10, TimeUnit.SECONDS);

    requestComplete.await(10, TimeUnit.SECONDS);
    assertStatus(client, location, "queued", 0, 1, 1, 1, "not scheduled");

    client.close();
    assertStatus(client, location, "closed", 0, 1, 1, 1, "not scheduled");
}
 

@Test
public void testCloseDuringPendingRequest()
        throws Exception
{
    CyclicBarrier beforeRequest = new CyclicBarrier(2);
    CyclicBarrier afterRequest = new CyclicBarrier(2);
    StaticRequestProcessor processor = new StaticRequestProcessor(beforeRequest, afterRequest);
    processor.setResponse(new TestingResponse(HttpStatus.NO_CONTENT, ImmutableListMultimap.<String, String>of(), new byte[0]));

    CyclicBarrier requestComplete = new CyclicBarrier(2);
    TestingClientCallback callback = new TestingClientCallback(requestComplete);

    URI location = URI.create("http://localhost:8080");
    HttpPageBufferClient client = new HttpPageBufferClient(new TestingHttpClient(processor, executor),
            new DataSize(10, Unit.MEGABYTE),
            new Duration(1, TimeUnit.MINUTES),
            location,
            callback,
            blockEncodingManager,
            executor,
            Stopwatch.createUnstarted());

    assertStatus(client, location, "queued", 0, 0, 0, 0, "not scheduled");

    // send request
    client.scheduleRequest();
    beforeRequest.await(10, TimeUnit.SECONDS);
    assertStatus(client, location, "running", 0, 1, 0, 0, "PROCESSING_REQUEST");
    assertEquals(client.isRunning(), true);
    // request is pending, now close it
    client.close();

    try {
        requestComplete.await(10, TimeUnit.SECONDS);
    }
    catch (BrokenBarrierException ignored) {
    }
    assertStatus(client, location, "closed", 0, 1, 1, 1, "not scheduled");
}
 

@Test
public void testWrapWithCacheIfTinyStripes()
        throws IOException
{
    DataSize maxMergeDistance = new DataSize(1, Unit.MEGABYTE);
    DataSize maxReadSize = new DataSize(8, Unit.MEGABYTE);

    OrcDataSource actual = wrapWithCacheIfTinyStripes(
            FakeOrcDataSource.INSTANCE,
            ImmutableList.of(),
            maxMergeDistance,
            maxReadSize);
    assertInstanceOf(actual, CachingOrcDataSource.class);

    actual = wrapWithCacheIfTinyStripes(
            FakeOrcDataSource.INSTANCE,
            ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10)),
            maxMergeDistance,
            maxReadSize);
    assertInstanceOf(actual, CachingOrcDataSource.class);

    actual = wrapWithCacheIfTinyStripes(
            FakeOrcDataSource.INSTANCE,
            ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10), new StripeInformation(123, 33, 10, 10, 10), new StripeInformation(123, 63, 10, 10, 10)),
            maxMergeDistance,
            maxReadSize);
    assertInstanceOf(actual, CachingOrcDataSource.class);

    actual = wrapWithCacheIfTinyStripes(
            FakeOrcDataSource.INSTANCE,
            ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10), new StripeInformation(123, 33, 10, 10, 10), new StripeInformation(123, 63, 1048576 * 8 - 20, 10, 10)),
            maxMergeDistance,
            maxReadSize);
    assertInstanceOf(actual, CachingOrcDataSource.class);

    actual = wrapWithCacheIfTinyStripes(
            FakeOrcDataSource.INSTANCE,
            ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10), new StripeInformation(123, 33, 10, 10, 10), new StripeInformation(123, 63, 1048576 * 8 - 20 + 1, 10, 10)),
            maxMergeDistance,
            maxReadSize);
    assertNotInstanceOf(actual, CachingOrcDataSource.class);
}
 

@Test
public void testTinyStripesReadCacheAt()
        throws IOException
{
    DataSize maxMergeDistance = new DataSize(1, Unit.MEGABYTE);
    DataSize maxReadSize = new DataSize(8, Unit.MEGABYTE);

    TestingOrcDataSource testingOrcDataSource = new TestingOrcDataSource(FakeOrcDataSource.INSTANCE);
    CachingOrcDataSource cachingOrcDataSource = new CachingOrcDataSource(
            testingOrcDataSource,
            createTinyStripesRangeFinder(
                    ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10), new StripeInformation(123, 33, 10, 10, 10), new StripeInformation(123, 63, 1048576 * 8 - 20, 10, 10)),
                    maxMergeDistance,
                    maxReadSize));
    cachingOrcDataSource.readCacheAt(3);
    assertEquals(testingOrcDataSource.getLastReadRanges(), ImmutableList.of(new DiskRange(3, 60)));
    cachingOrcDataSource.readCacheAt(63);
    assertEquals(testingOrcDataSource.getLastReadRanges(), ImmutableList.of(new DiskRange(63, 8 * 1048576)));

    testingOrcDataSource = new TestingOrcDataSource(FakeOrcDataSource.INSTANCE);
    cachingOrcDataSource = new CachingOrcDataSource(
            testingOrcDataSource,
            createTinyStripesRangeFinder(
                    ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10), new StripeInformation(123, 33, 10, 10, 10), new StripeInformation(123, 63, 1048576 * 8 - 20, 10, 10)),
                    maxMergeDistance,
                    maxReadSize));
    cachingOrcDataSource.readCacheAt(62); // read at the end of a stripe
    assertEquals(testingOrcDataSource.getLastReadRanges(), ImmutableList.of(new DiskRange(3, 60)));
    cachingOrcDataSource.readCacheAt(63);
    assertEquals(testingOrcDataSource.getLastReadRanges(), ImmutableList.of(new DiskRange(63, 8 * 1048576)));

    testingOrcDataSource = new TestingOrcDataSource(FakeOrcDataSource.INSTANCE);
    cachingOrcDataSource = new CachingOrcDataSource(
            testingOrcDataSource,
            createTinyStripesRangeFinder(
                    ImmutableList.of(new StripeInformation(123, 3, 1, 0, 0), new StripeInformation(123, 4, 1048576, 1048576, 1048576 * 3), new StripeInformation(123, 4 + 1048576 * 5, 1048576, 1048576, 1048576)),
                    maxMergeDistance,
                    maxReadSize));
    cachingOrcDataSource.readCacheAt(3);
    assertEquals(testingOrcDataSource.getLastReadRanges(), ImmutableList.of(new DiskRange(3, 1 + 1048576 * 5)));
    cachingOrcDataSource.readCacheAt(4 + 1048576 * 5);
    assertEquals(testingOrcDataSource.getLastReadRanges(), ImmutableList.of(new DiskRange(4 + 1048576 * 5, 3 * 1048576)));
}
 

@Test(timeOut = 10000)
public void testAddLocation()
        throws Exception
{
    DataSize maxResponseSize = new DataSize(10, Unit.MEGABYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);

    @SuppressWarnings("resource")
    ExchangeClient exchangeClient = new ExchangeClient(blockEncodingManager, new DataSize(32, Unit.MEGABYTE), maxResponseSize, 1, new Duration(1, TimeUnit.MINUTES), new TestingHttpClient(processor, newCachedThreadPool(daemonThreadsNamed("test-%s"))), executor, deltaMemoryInBytes -> { });

    URI location1 = URI.create("http://localhost:8081/foo");
    processor.addPage(location1, createPage(1));
    processor.addPage(location1, createPage(2));
    processor.addPage(location1, createPage(3));
    processor.setComplete(location1);
    exchangeClient.addLocation(location1);

    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(1));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(2));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(3));

    assertNull(exchangeClient.getNextPage(new Duration(10, TimeUnit.MILLISECONDS)));
    assertEquals(exchangeClient.isClosed(), false);

    URI location2 = URI.create("http://localhost:8082/bar");
    processor.addPage(location2, createPage(4));
    processor.addPage(location2, createPage(5));
    processor.addPage(location2, createPage(6));
    processor.setComplete(location2);
    exchangeClient.addLocation(location2);

    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(4));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(5));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(exchangeClient.getNextPage(new Duration(1, TimeUnit.SECONDS)), createPage(6));

    assertNull(exchangeClient.getNextPage(new Duration(10, TimeUnit.MILLISECONDS)));
    assertEquals(exchangeClient.isClosed(), false);

    exchangeClient.noMoreLocations();
    // The transition to closed may happen asynchronously, since it requires that all the HTTP clients
    // receive a final GONE response, so just spin until it's closed or the test times out.
    while (!exchangeClient.isClosed()) {
        Thread.sleep(1);
    }

    ImmutableMap<URI, PageBufferClientStatus> statuses = uniqueIndex(exchangeClient.getStatus().getPageBufferClientStatuses(), PageBufferClientStatus::getUri);
    assertStatus(statuses.get(location1), location1, "closed", 3, 3, 3, "not scheduled");
    assertStatus(statuses.get(location2), location2, "closed", 3, 3, 3, "not scheduled");
}
 

@Test
public void testInvalidResponses()
        throws Exception
{
    CyclicBarrier beforeRequest = new CyclicBarrier(1);
    CyclicBarrier afterRequest = new CyclicBarrier(1);
    StaticRequestProcessor processor = new StaticRequestProcessor(beforeRequest, afterRequest);

    CyclicBarrier requestComplete = new CyclicBarrier(2);
    TestingClientCallback callback = new TestingClientCallback(requestComplete);

    URI location = URI.create("http://localhost:8080");
    HttpPageBufferClient client = new HttpPageBufferClient(new TestingHttpClient(processor, executor),
            new DataSize(10, Unit.MEGABYTE),
            new Duration(1, TimeUnit.MINUTES),
            location,
            callback,
            blockEncodingManager,
            executor,
            Stopwatch.createUnstarted());

    assertStatus(client, location, "queued", 0, 0, 0, 0, "not scheduled");

    // send not found response and verify response was ignored
    processor.setResponse(new TestingResponse(HttpStatus.NOT_FOUND, ImmutableListMultimap.of(CONTENT_TYPE, PRESTO_PAGES), new byte[0]));
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 1);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 1);
    assertInstanceOf(callback.getFailure(), PageTransportErrorException.class);
    assertContains(callback.getFailure().getMessage(), "Expected response code to be 200, but was 404 Not Found");
    assertStatus(client, location, "queued", 0, 1, 1, 1, "not scheduled");

    // send invalid content type response and verify response was ignored
    callback.resetStats();
    processor.setResponse(new TestingResponse(HttpStatus.OK, ImmutableListMultimap.of(CONTENT_TYPE, "INVALID_TYPE"), new byte[0]));
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 1);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 1);
    assertInstanceOf(callback.getFailure(), PageTransportErrorException.class);
    assertContains(callback.getFailure().getMessage(), "Expected application/x-presto-pages response from server but got INVALID_TYPE");
    assertStatus(client, location, "queued", 0, 2, 2, 2, "not scheduled");

    // send unexpected content type response and verify response was ignored
    callback.resetStats();
    processor.setResponse(new TestingResponse(HttpStatus.OK, ImmutableListMultimap.of(CONTENT_TYPE, "text/plain"), new byte[0]));
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 1);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 1);
    assertInstanceOf(callback.getFailure(), PageTransportErrorException.class);
    assertContains(callback.getFailure().getMessage(), "Expected application/x-presto-pages response from server but got text/plain");
    assertStatus(client, location, "queued", 0, 3, 3, 3, "not scheduled");

    // close client and verify
    client.close();
    assertStatus(client, location, "closed", 0, 3, 3, 3, "not scheduled");
}
 

@Test
public void testExceptionFromResponseHandler()
        throws Exception
{
    TestingTicker ticker = new TestingTicker();
    AtomicReference<Duration> tickerIncrement = new AtomicReference<>(new Duration(0, TimeUnit.SECONDS));

    TestingHttpClient.Processor processor = (input) -> {
        Duration delta = tickerIncrement.get();
        ticker.increment(delta.toMillis(), TimeUnit.MILLISECONDS);
        throw new RuntimeException("Foo");
    };

    CyclicBarrier requestComplete = new CyclicBarrier(2);
    TestingClientCallback callback = new TestingClientCallback(requestComplete);

    URI location = URI.create("http://localhost:8080");
    HttpPageBufferClient client = new HttpPageBufferClient(new TestingHttpClient(processor, executor),
            new DataSize(10, Unit.MEGABYTE),
            new Duration(1, TimeUnit.MINUTES),
            location,
            callback,
            blockEncodingManager,
            executor,
            Stopwatch.createUnstarted(ticker));

    assertStatus(client, location, "queued", 0, 0, 0, 0, "not scheduled");

    // request processor will throw exception, verify the request is marked a completed
    // this starts the error stopwatch
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 1);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 0);
    assertStatus(client, location, "queued", 0, 1, 1, 1, "not scheduled");

    // advance time forward, but not enough to fail the client
    tickerIncrement.set(new Duration(30, TimeUnit.SECONDS));

    // verify that the client has not failed
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 2);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 0);
    assertStatus(client, location, "queued", 0, 2, 2, 2, "not scheduled");

    // advance time forward beyond the minimum error duration
    tickerIncrement.set(new Duration(31, TimeUnit.SECONDS));

    // verify that the client has failed
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 3);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 1);
    assertInstanceOf(callback.getFailure(), PageTransportTimeoutException.class);
    assertContains(callback.getFailure().getMessage(), WORKER_NODE_ERROR + " (http://localhost:8080/0 - requests failed for 61.00s)");
    assertStatus(client, location, "queued", 0, 3, 3, 3, "not scheduled");
}
 

@Test
public void testWrapWithCacheIfTinyStripes()
        throws IOException
{
    DataSize maxMergeDistance = new DataSize(1, Unit.MEGABYTE);
    DataSize maxReadSize = new DataSize(8, Unit.MEGABYTE);

    OrcDataSource actual = wrapWithCacheIfTinyStripes(
            FakeOrcDataSource.INSTANCE,
            ImmutableList.of(),
            maxMergeDistance,
            maxReadSize);
    Assert.assertTrue(actual instanceof CachingOrcDataSource);

    actual = wrapWithCacheIfTinyStripes(
            FakeOrcDataSource.INSTANCE,
            ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10)),
            maxMergeDistance,
            maxReadSize);
    Assert.assertTrue(actual instanceof CachingOrcDataSource);

    actual = wrapWithCacheIfTinyStripes(
            FakeOrcDataSource.INSTANCE,
            ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10), new StripeInformation(123, 33, 10, 10, 10), new StripeInformation(123, 63, 10, 10, 10)),
            maxMergeDistance,
            maxReadSize);
    Assert.assertTrue(actual instanceof CachingOrcDataSource);

    actual = wrapWithCacheIfTinyStripes(
            FakeOrcDataSource.INSTANCE,
            ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10), new StripeInformation(123, 33, 10, 10, 10), new StripeInformation(123, 63, 1048576 * 8 - 20, 10, 10)),
            maxMergeDistance,
            maxReadSize);
    Assert.assertTrue(actual instanceof CachingOrcDataSource);

    actual = wrapWithCacheIfTinyStripes(
            FakeOrcDataSource.INSTANCE,
            ImmutableList.of(new StripeInformation(123, 3, 10, 10, 10), new StripeInformation(123, 33, 10, 10, 10), new StripeInformation(123, 63, 1048576 * 8 - 20 + 1, 10, 10)),
            maxMergeDistance,
            maxReadSize);
    assertNotInstanceOf(actual, CachingOrcDataSource.class);
}