Java BlobStore类代码示例

import org.elasticsearch.common.blobstore.BlobStore; //导入依赖的package包/类
public void testContainerCreationAndDeletion() throws IOException {
    try(BlobStore store = newBlobStore()) {
        final BlobContainer containerFoo = store.blobContainer(new BlobPath().add("foo"));
        final BlobContainer containerBar = store.blobContainer(new BlobPath().add("bar"));
        byte[] data1 = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
        byte[] data2 = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
        writeBlob(containerFoo, "test", new BytesArray(data1));
        writeBlob(containerBar, "test", new BytesArray(data2));

        assertArrayEquals(readBlobFully(containerFoo, "test", data1.length), data1);
        assertArrayEquals(readBlobFully(containerBar, "test", data2.length), data2);

        assertTrue(containerFoo.blobExists("test"));
        assertTrue(containerBar.blobExists("test"));
        store.delete(new BlobPath());
        assertFalse(containerFoo.blobExists("test"));
        assertFalse(containerBar.blobExists("test"));
    }
}
 

import org.elasticsearch.common.blobstore.BlobStore; //导入依赖的package包/类
public void testWriteRead() throws IOException {
    try(BlobStore store = newBlobStore()) {
        final BlobContainer container = store.blobContainer(new BlobPath());
        byte[] data = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
        writeBlob(container, "foobar", new BytesArray(data));
        try (InputStream stream = container.readBlob("foobar")) {
            BytesRefBuilder target = new BytesRefBuilder();
            while (target.length() < data.length) {
                byte[] buffer = new byte[scaledRandomIntBetween(1, data.length - target.length())];
                int offset = scaledRandomIntBetween(0, buffer.length - 1);
                int read = stream.read(buffer, offset, buffer.length - offset);
                target.append(new BytesRef(buffer, offset, read));
            }
            assertEquals(data.length, target.length());
            assertArrayEquals(data, Arrays.copyOfRange(target.bytes(), 0, target.length()));
        }
    }
}
 

import org.elasticsearch.common.blobstore.BlobStore; //导入依赖的package包/类
public void testBlobStoreOperations() throws IOException {
    BlobStore blobStore = createTestBlobStore();
    BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
    ChecksumBlobStoreFormat<BlobObj> checksumJSON = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
        xContentRegistry(), false, XContentType.JSON);
    ChecksumBlobStoreFormat<BlobObj> checksumSMILE = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
        xContentRegistry(), false, XContentType.SMILE);
    ChecksumBlobStoreFormat<BlobObj> checksumSMILECompressed = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
        xContentRegistry(), true, XContentType.SMILE);

    // Write blobs in different formats
    checksumJSON.write(new BlobObj("checksum json"), blobContainer, "check-json");
    checksumSMILE.write(new BlobObj("checksum smile"), blobContainer, "check-smile");
    checksumSMILECompressed.write(new BlobObj("checksum smile compressed"), blobContainer, "check-smile-comp");

    // Assert that all checksum blobs can be read by all formats
    assertEquals(checksumJSON.read(blobContainer, "check-json").getText(), "checksum json");
    assertEquals(checksumSMILE.read(blobContainer, "check-json").getText(), "checksum json");
    assertEquals(checksumJSON.read(blobContainer, "check-smile").getText(), "checksum smile");
    assertEquals(checksumSMILE.read(blobContainer, "check-smile").getText(), "checksum smile");
    assertEquals(checksumJSON.read(blobContainer, "check-smile-comp").getText(), "checksum smile compressed");
    assertEquals(checksumSMILE.read(blobContainer, "check-smile-comp").getText(), "checksum smile compressed");
}
 

import org.elasticsearch.common.blobstore.BlobStore; //导入依赖的package包/类
public void testCompressionIsApplied() throws IOException {
    BlobStore blobStore = createTestBlobStore();
    BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
    StringBuilder veryRedundantText = new StringBuilder();
    for (int i = 0; i < randomIntBetween(100, 300); i++) {
        veryRedundantText.append("Blah ");
    }
    ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
        xContentRegistry(), false, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
    ChecksumBlobStoreFormat<BlobObj> checksumFormatComp = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
        xContentRegistry(), true, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
    BlobObj blobObj = new BlobObj(veryRedundantText.toString());
    checksumFormatComp.write(blobObj, blobContainer, "blob-comp");
    checksumFormat.write(blobObj, blobContainer, "blob-not-comp");
    Map<String, BlobMetaData> blobs = blobContainer.listBlobsByPrefix("blob-");
    assertEquals(blobs.size(), 2);
    assertThat(blobs.get("blob-not-comp").length(), greaterThan(blobs.get("blob-comp").length()));
}
 

import org.elasticsearch.common.blobstore.BlobStore; //导入依赖的package包/类
public void testDeleteBlob() throws IOException {
    try (BlobStore store = newBlobStore()) {
        final String blobName = "foobar";
        final BlobContainer container = store.blobContainer(new BlobPath());
        expectThrows(IOException.class, () -> container.deleteBlob(blobName));

        byte[] data = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
        final BytesArray bytesArray = new BytesArray(data);
        writeBlob(container, blobName, bytesArray);
        container.deleteBlob(blobName); // should not raise

        // blob deleted, so should raise again
        expectThrows(IOException.class, () -> container.deleteBlob(blobName));
    }
}
 

import org.elasticsearch.common.blobstore.BlobStore; //导入依赖的package包/类
public void testVerifyOverwriteFails() throws IOException {
    try (BlobStore store = newBlobStore()) {
        final String blobName = "foobar";
        final BlobContainer container = store.blobContainer(new BlobPath());
        byte[] data = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
        final BytesArray bytesArray = new BytesArray(data);
        writeBlob(container, blobName, bytesArray);
        // should not be able to overwrite existing blob
        expectThrows(IOException.class, () -> writeBlob(container, blobName, bytesArray));
        container.deleteBlob(blobName);
        writeBlob(container, blobName, bytesArray); // after deleting the previous blob, we should be able to write to it again
    }
}
 

import org.elasticsearch.common.blobstore.BlobStore; //导入依赖的package包/类
public void testAtomicWrite() throws Exception {
    final BlobStore blobStore = createTestBlobStore();
    final BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
    String testString = randomAsciiOfLength(randomInt(10000));
    final CountDownLatch block = new CountDownLatch(1);
    final CountDownLatch unblock = new CountDownLatch(1);
    final BlobObj blobObj = new BlobObj(testString) {
        @Override
        public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
            super.toXContent(builder, params);
            // Block before finishing writing
            try {
                block.countDown();
                unblock.await(5, TimeUnit.SECONDS);
            } catch (InterruptedException ex) {
                Thread.currentThread().interrupt();
            }
            return builder;
        }
    };
    final ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
        xContentRegistry(), randomBoolean(), randomBoolean() ? XContentType.SMILE : XContentType.JSON);
    ExecutorService threadPool = Executors.newFixedThreadPool(1);
    try {
        Future<Void> future = threadPool.submit(new Callable<Void>() {
            @Override
            public Void call() throws Exception {
                checksumFormat.writeAtomic(blobObj, blobContainer, "test-blob");
                return null;
            }
        });
        block.await(5, TimeUnit.SECONDS);
        assertFalse(blobContainer.blobExists("test-blob"));
        unblock.countDown();
        future.get();
        assertTrue(blobContainer.blobExists("test-blob"));
    } finally {
        threadPool.shutdown();
    }
}
 

import org.elasticsearch.common.blobstore.BlobStore; //导入依赖的package包/类
@Override
protected BlobStore newBlobStore() throws IOException {
    try {
        RepositoryMetaData repositoryMetaData = new RepositoryMetaData("azure", "ittest", Settings.EMPTY);
        AzureStorageServiceMock client = new AzureStorageServiceMock();
        return new AzureBlobStore(repositoryMetaData, Settings.EMPTY, client);
    } catch (URISyntaxException | StorageException e) {
        throw new IOException(e);
    }
}
 

import org.elasticsearch.common.blobstore.BlobStore; //导入依赖的package包/类
protected BlobStore newBlobStore() throws IOException {
    MockAmazonS3 client = new MockAmazonS3();
    String bucket = randomAsciiOfLength(randomIntBetween(1, 10)).toLowerCase(Locale.ROOT);

    return new S3BlobStore(Settings.EMPTY, client, bucket, false,
        new ByteSizeValue(10, ByteSizeUnit.MB), 5, "public-read-write", "standard");
}
 

import org.elasticsearch.common.blobstore.BlobStore; //导入依赖的package包/类
@Override
protected BlobStore newBlobStore() throws IOException {
    FileContext fileContext;
    try {
        fileContext = AccessController.doPrivileged((PrivilegedExceptionAction<FileContext>)
            () -> createContext(new URI("hdfs:///")));
    } catch (PrivilegedActionException e) {
        throw new RuntimeException(e.getCause());
    }
    return new HdfsBlobStore(fileContext, "temp", 1024);
}
 

import org.elasticsearch.common.blobstore.BlobStore; //导入依赖的package包/类
/**
 * Called by {@link org.elasticsearch.repositories.blobstore.BlobStoreRepository} on repository startup
 *
 * @param blobStore blob store
 * @param basePath  base path to blob store
 * @param chunkSize chunk size
 */
public void initialize(BlobStore blobStore, BlobPath basePath, ByteSizeValue chunkSize,
                       RateLimiter snapshotRateLimiter, RateLimiter restoreRateLimiter,
                       final RateLimiterListener rateLimiterListener, boolean compress) {
    this.blobStore = blobStore;
    this.basePath = basePath;
    this.chunkSize = chunkSize;
    this.snapshotRateLimiter = snapshotRateLimiter;
    this.restoreRateLimiter = restoreRateLimiter;
    this.rateLimiterListener = rateLimiterListener;
    this.snapshotThrottleListener = new RateLimitingInputStream.Listener() {
        @Override
        public void onPause(long nanos) {
            rateLimiterListener.onSnapshotPause(nanos);
        }
    };
    this.restoreThrottleListener = new RateLimitingInputStream.Listener() {
        @Override
        public void onPause(long nanos) {
            rateLimiterListener.onRestorePause(nanos);
        }
    };
    this.compress = compress;
    indexShardSnapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT, BlobStoreIndexShardSnapshot.PROTO, parseFieldMatcher, isCompress());
    indexShardSnapshotLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_SNAPSHOT_NAME_FORMAT, BlobStoreIndexShardSnapshot.PROTO, parseFieldMatcher);
    indexShardSnapshotsFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_INDEX_CODEC, SNAPSHOT_INDEX_NAME_FORMAT, BlobStoreIndexShardSnapshots.PROTO, parseFieldMatcher, isCompress());
}
 

import org.elasticsearch.common.blobstore.BlobStore; //导入依赖的package包/类
public CloudFilesBlobStore(Settings settings, RegionScopedBlobStoreContext context, String container, Location location, Executor executor) {
    super(settings);
    this.blobStoreContext = context;
    this.container = container;
    this.executor = executor;
    this.location = location;

    this.bufferSizeInBytes = (int) settings.getAsBytesSize("buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).bytes();

    // create the cloud files container if it does not already exist
    org.jclouds.blobstore.BlobStore blobStore = blobStoreContext.getBlobStore(location.getId());
    blobStore.createContainerInLocation(getOnlyElement(blobStore.listAssignableLocations()), container);
}
 

import org.elasticsearch.common.blobstore.BlobStore; //导入依赖的package包/类
@Override
protected BlobStore blobStore() {
    return blobStore;
}
 

import org.elasticsearch.common.blobstore.BlobStore; //导入依赖的package包/类
protected BlobStore createTestBlobStore() throws IOException {
    Settings settings = Settings.builder().build();
    return new FsBlobStore(settings, randomRepoPath());
}
 

import org.elasticsearch.common.blobstore.BlobStore; //导入依赖的package包/类
public BlobStoreWrapper(BlobStore delegate) {
    this.delegate = delegate;
}