Java BlobContainer类代码示例

import org.elasticsearch.common.blobstore.BlobContainer; //导入依赖的package包/类
public void testContainerCreationAndDeletion() throws IOException {
    try(BlobStore store = newBlobStore()) {
        final BlobContainer containerFoo = store.blobContainer(new BlobPath().add("foo"));
        final BlobContainer containerBar = store.blobContainer(new BlobPath().add("bar"));
        byte[] data1 = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
        byte[] data2 = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
        writeBlob(containerFoo, "test", new BytesArray(data1));
        writeBlob(containerBar, "test", new BytesArray(data2));

        assertArrayEquals(readBlobFully(containerFoo, "test", data1.length), data1);
        assertArrayEquals(readBlobFully(containerBar, "test", data2.length), data2);

        assertTrue(containerFoo.blobExists("test"));
        assertTrue(containerBar.blobExists("test"));
        store.delete(new BlobPath());
        assertFalse(containerFoo.blobExists("test"));
        assertFalse(containerBar.blobExists("test"));
    }
}
 

import org.elasticsearch.common.blobstore.BlobContainer; //导入依赖的package包/类
public void testWriteRead() throws IOException {
    try(BlobStore store = newBlobStore()) {
        final BlobContainer container = store.blobContainer(new BlobPath());
        byte[] data = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16)));
        writeBlob(container, "foobar", new BytesArray(data));
        try (InputStream stream = container.readBlob("foobar")) {
            BytesRefBuilder target = new BytesRefBuilder();
            while (target.length() < data.length) {
                byte[] buffer = new byte[scaledRandomIntBetween(1, data.length - target.length())];
                int offset = scaledRandomIntBetween(0, buffer.length - 1);
                int read = stream.read(buffer, offset, buffer.length - offset);
                target.append(new BytesRef(buffer, offset, read));
            }
            assertEquals(data.length, target.length());
            assertArrayEquals(data, Arrays.copyOfRange(target.bytes(), 0, target.length()));
        }
    }
}
 

import org.elasticsearch.common.blobstore.BlobContainer; //导入依赖的package包/类
/**
 * Reads blob with specified name without resolving the blobName using using {@link #blobName} method.
 *
 * @param blobContainer blob container
 * @param blobName blob name
 */
public T readBlob(BlobContainer blobContainer, String blobName) throws IOException {
    try (InputStream inputStream = blobContainer.readBlob(blobName)) {
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        Streams.copy(inputStream, out);
        final byte[] bytes = out.toByteArray();
        final String resourceDesc = "ChecksumBlobStoreFormat.readBlob(blob=\"" + blobName + "\")";
        try (ByteArrayIndexInput indexInput = new ByteArrayIndexInput(resourceDesc, bytes)) {
            CodecUtil.checksumEntireFile(indexInput);
            CodecUtil.checkHeader(indexInput, codec, VERSION, VERSION);
            long filePointer = indexInput.getFilePointer();
            long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
            BytesReference bytesReference = new BytesArray(bytes, (int) filePointer, (int) contentSize);
            return read(bytesReference);
        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
            // we trick this into a dedicated exception with the original stacktrace
            throw new CorruptStateException(ex);
        }
    }
}
 

import org.elasticsearch.common.blobstore.BlobContainer; //导入依赖的package包/类
/**
 * Writes blob in atomic manner without resolving the blobName using using {@link #blobName} method.
 * <p>
 * The blob will be compressed and checksum will be written if required.
 *
 * @param obj           object to be serialized
 * @param blobContainer blob container
 * @param blobName          blob name
 */
protected void writeBlob(T obj, BlobContainer blobContainer, String blobName) throws IOException {
    BytesReference bytes = write(obj);
    try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
        final String resourceDesc = "ChecksumBlobStoreFormat.writeBlob(blob=\"" + blobName + "\")";
        try (OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput(resourceDesc, blobName, byteArrayOutputStream, BUFFER_SIZE)) {
            CodecUtil.writeHeader(indexOutput, codec, VERSION);
            try (OutputStream indexOutputOutputStream = new IndexOutputOutputStream(indexOutput) {
                @Override
                public void close() throws IOException {
                    // this is important since some of the XContentBuilders write bytes on close.
                    // in order to write the footer we need to prevent closing the actual index input.
                } }) {
                bytes.writeTo(indexOutputOutputStream);
            }
            CodecUtil.writeFooter(indexOutput);
        }
        BytesArray bytesArray = new BytesArray(byteArrayOutputStream.toByteArray());
        try (InputStream stream = bytesArray.streamInput()) {
            blobContainer.writeBlob(blobName, stream, bytesArray.length());
        }
    }
}
 

import org.elasticsearch.common.blobstore.BlobContainer; //导入依赖的package包/类
@Override
public String startVerification() {
    try {
        if (isReadOnly()) {
            // It's readonly - so there is not much we can do here to verify it
            return null;
        } else {
            String seed = UUIDs.randomBase64UUID();
            byte[] testBytes = Strings.toUTF8Bytes(seed);
            BlobContainer testContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed)));
            String blobName = "master.dat";
            BytesArray bytes = new BytesArray(testBytes);
            try (InputStream stream = bytes.streamInput()) {
                testContainer.writeBlob(blobName + "-temp", stream, bytes.length());
            }
            // Make sure that move is supported
            testContainer.move(blobName + "-temp", blobName);
            return seed;
        }
    } catch (IOException exp) {
        throw new RepositoryVerificationException(metadata.name(), "path " + basePath() + " is not accessible on master node", exp);
    }
}
 

import org.elasticsearch.common.blobstore.BlobContainer; //导入依赖的package包/类
@Override
public void verify(String seed, DiscoveryNode localNode) {
    BlobContainer testBlobContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed)));
    if (testBlobContainer.blobExists("master.dat")) {
        try  {
            BytesArray bytes = new BytesArray(seed);
            try (InputStream stream = bytes.streamInput()) {
                testBlobContainer.writeBlob("data-" + localNode.getId() + ".dat", stream, bytes.length());
            }
        } catch (IOException exp) {
            throw new RepositoryVerificationException(metadata.name(), "store location [" + blobStore() + "] is not accessible on the node [" + localNode + "]", exp);
        }
    } else {
        throw new RepositoryVerificationException(metadata.name(), "a file written by master to the store [" + blobStore() + "] cannot be accessed on the node [" + localNode + "]. "
            + "This might indicate that the store [" + blobStore() + "] is not shared between this node and the master node or "
            + "that permissions on the store don't allow reading files written by the master node");
    }
}
 

import org.elasticsearch.common.blobstore.BlobContainer; //导入依赖的package包/类
/**
 * This is a BWC layer to ensure we update the snapshots metadata with the corresponding hashes before we compare them.
 * The new logic for StoreFileMetaData reads the entire <tt>.si</tt> and <tt>segments.n</tt> files to strengthen the
 * comparison of the files on a per-segment / per-commit level.
 */
private static void maybeRecalculateMetadataHash(final BlobContainer blobContainer, final BlobStoreIndexShardSnapshot.FileInfo fileInfo, Store.MetadataSnapshot snapshot) throws Exception {
    final StoreFileMetaData metadata;
    if (fileInfo != null && (metadata = snapshot.get(fileInfo.physicalName())) != null) {
        if (metadata.hash().length > 0 && fileInfo.metadata().hash().length == 0) {
            // we have a hash - check if our repo has a hash too otherwise we have
            // to calculate it.
            // we might have multiple parts even though the file is small... make sure we read all of it.
            try (InputStream stream = new PartSliceStream(blobContainer, fileInfo)) {
                BytesRefBuilder builder = new BytesRefBuilder();
                Store.MetadataSnapshot.hashFile(builder, stream, fileInfo.length());
                BytesRef hash = fileInfo.metadata().hash(); // reset the file infos metadata hash
                assert hash.length == 0;
                hash.bytes = builder.bytes();
                hash.offset = 0;
                hash.length = builder.length();
            }
        }
    }
}
 

import org.elasticsearch.common.blobstore.BlobContainer; //导入依赖的package包/类
public void testBlobStoreOperations() throws IOException {
    BlobStore blobStore = createTestBlobStore();
    BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
    ChecksumBlobStoreFormat<BlobObj> checksumJSON = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
        xContentRegistry(), false, XContentType.JSON);
    ChecksumBlobStoreFormat<BlobObj> checksumSMILE = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
        xContentRegistry(), false, XContentType.SMILE);
    ChecksumBlobStoreFormat<BlobObj> checksumSMILECompressed = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
        xContentRegistry(), true, XContentType.SMILE);

    // Write blobs in different formats
    checksumJSON.write(new BlobObj("checksum json"), blobContainer, "check-json");
    checksumSMILE.write(new BlobObj("checksum smile"), blobContainer, "check-smile");
    checksumSMILECompressed.write(new BlobObj("checksum smile compressed"), blobContainer, "check-smile-comp");

    // Assert that all checksum blobs can be read by all formats
    assertEquals(checksumJSON.read(blobContainer, "check-json").getText(), "checksum json");
    assertEquals(checksumSMILE.read(blobContainer, "check-json").getText(), "checksum json");
    assertEquals(checksumJSON.read(blobContainer, "check-smile").getText(), "checksum smile");
    assertEquals(checksumSMILE.read(blobContainer, "check-smile").getText(), "checksum smile");
    assertEquals(checksumJSON.read(blobContainer, "check-smile-comp").getText(), "checksum smile compressed");
    assertEquals(checksumSMILE.read(blobContainer, "check-smile-comp").getText(), "checksum smile compressed");
}
 

import org.elasticsearch.common.blobstore.BlobContainer; //导入依赖的package包/类
public void testCompressionIsApplied() throws IOException {
    BlobStore blobStore = createTestBlobStore();
    BlobContainer blobContainer = blobStore.blobContainer(BlobPath.cleanPath());
    StringBuilder veryRedundantText = new StringBuilder();
    for (int i = 0; i < randomIntBetween(100, 300); i++) {
        veryRedundantText.append("Blah ");
    }
    ChecksumBlobStoreFormat<BlobObj> checksumFormat = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
        xContentRegistry(), false, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
    ChecksumBlobStoreFormat<BlobObj> checksumFormatComp = new ChecksumBlobStoreFormat<>(BLOB_CODEC, "%s", BlobObj::fromXContent,
        xContentRegistry(), true, randomBoolean() ? XContentType.SMILE : XContentType.JSON);
    BlobObj blobObj = new BlobObj(veryRedundantText.toString());
    checksumFormatComp.write(blobObj, blobContainer, "blob-comp");
    checksumFormat.write(blobObj, blobContainer, "blob-not-comp");
    Map<String, BlobMetaData> blobs = blobContainer.listBlobsByPrefix("blob-");
    assertEquals(blobs.size(), 2);
    assertThat(blobs.get("blob-not-comp").length(), greaterThan(blobs.get("blob-comp").length()));
}
 

import org.elasticsearch.common.blobstore.BlobContainer; //导入依赖的package包/类
protected void randomCorruption(BlobContainer blobContainer, String blobName) throws IOException {
    byte[] buffer = new byte[(int) blobContainer.listBlobsByPrefix(blobName).get(blobName).length()];
    long originalChecksum = checksum(buffer);
    try (InputStream inputStream = blobContainer.readBlob(blobName)) {
        Streams.readFully(inputStream, buffer);
    }
    do {
        int location = randomIntBetween(0, buffer.length - 1);
        buffer[location] = (byte) (buffer[location] ^ 42);
    } while (originalChecksum == checksum(buffer));
    blobContainer.deleteBlob(blobName); // delete original before writing new blob
    BytesArray bytesArray = new BytesArray(buffer);
    try (StreamInput stream = bytesArray.streamInput()) {
        blobContainer.writeBlob(blobName, stream, bytesArray.length());
    }
}
 

import org.elasticsearch.common.blobstore.BlobContainer; //导入依赖的package包/类
/**
 * Reads blob with specified name without resolving the blobName using using {@link #blobName} method.
 *
 * @param blobContainer blob container
 * @param blobName blob name
 */
public T readBlob(BlobContainer blobContainer, String blobName) throws IOException {
    try (InputStream inputStream = blobContainer.readBlob(blobName)) {
        byte[] bytes = ByteStreams.toByteArray(inputStream);
        final String resourceDesc = "ChecksumBlobStoreFormat.readBlob(blob=\"" + blobName + "\")";
        try (ByteArrayIndexInput indexInput = new ByteArrayIndexInput(resourceDesc, bytes)) {
            CodecUtil.checksumEntireFile(indexInput);
            CodecUtil.checkHeader(indexInput, codec, VERSION, VERSION);
            long filePointer = indexInput.getFilePointer();
            long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
            BytesReference bytesReference = new BytesArray(bytes, (int) filePointer, (int) contentSize);
            return read(bytesReference);
        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
            // we trick this into a dedicated exception with the original stacktrace
            throw new CorruptStateException(ex);
        }
    }
}
 

import org.elasticsearch.common.blobstore.BlobContainer; //导入依赖的package包/类
/**
 * Writes blob in atomic manner without resolving the blobName using using {@link #blobName} method.
 * <p>
 * The blob will be compressed and checksum will be written if required.
 *
 * @param obj           object to be serialized
 * @param blobContainer blob container
 * @param blobName          blob name
 */
protected void writeBlob(T obj, BlobContainer blobContainer, String blobName) throws IOException {
    BytesReference bytes = write(obj);
    try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream()) {
        final String resourceDesc = "ChecksumBlobStoreFormat.writeBlob(blob=\"" + blobName + "\")";
        try (OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput(resourceDesc, byteArrayOutputStream, BUFFER_SIZE)) {
            CodecUtil.writeHeader(indexOutput, codec, VERSION);
            try (OutputStream indexOutputOutputStream = new IndexOutputOutputStream(indexOutput) {
                @Override
                public void close() throws IOException {
                    // this is important since some of the XContentBuilders write bytes on close.
                    // in order to write the footer we need to prevent closing the actual index input.
                } }) {
                bytes.writeTo(indexOutputOutputStream);
            }
            CodecUtil.writeFooter(indexOutput);
        }
        blobContainer.writeBlob(blobName, new BytesArray(byteArrayOutputStream.toByteArray()));
    }
}
 

import org.elasticsearch.common.blobstore.BlobContainer; //导入依赖的package包/类
/**
 * {@inheritDoc}
 */
@Override
public void initializeSnapshot(SnapshotId snapshotId, List<String> indices, MetaData metaData) {
    if (readOnly()) {
        throw new RepositoryException(this.repositoryName, "cannot create snapshot in a readonly repository");
    }
    try {
        if (snapshotFormat.exists(snapshotsBlobContainer, snapshotId.getSnapshot()) ||
                snapshotLegacyFormat.exists(snapshotsBlobContainer, snapshotId.getSnapshot())) {
            throw new InvalidSnapshotNameException(snapshotId, "snapshot with such name already exists");
        }
        // Write Global MetaData
        globalMetaDataFormat.write(metaData, snapshotsBlobContainer, snapshotId.getSnapshot());
        for (String index : indices) {
            final IndexMetaData indexMetaData = metaData.index(index);
            final BlobPath indexPath = basePath().add("indices").add(index);
            final BlobContainer indexMetaDataBlobContainer = blobStore().blobContainer(indexPath);
            indexMetaDataFormat.write(indexMetaData, indexMetaDataBlobContainer, snapshotId.getSnapshot());
        }
    } catch (IOException ex) {
        throw new SnapshotCreationException(snapshotId, ex);
    }
}
 

import org.elasticsearch.common.blobstore.BlobContainer; //导入依赖的package包/类
@Override
public String startVerification() {
    try {
        if (readOnly()) {
            // It's readonly - so there is not much we can do here to verify it
            return null;
        } else {
            String seed = Strings.randomBase64UUID();
            byte[] testBytes = Strings.toUTF8Bytes(seed);
            BlobContainer testContainer = blobStore().blobContainer(basePath().add(testBlobPrefix(seed)));
            String blobName = "master.dat";
            testContainer.writeBlob(blobName + "-temp", new BytesArray(testBytes));
            // Make sure that move is supported
            testContainer.move(blobName + "-temp", blobName);
            return seed;
        }
    } catch (IOException exp) {
        throw new RepositoryVerificationException(repositoryName, "path " + basePath() + " is not accessible on master node", exp);
    }
}
 

import org.elasticsearch.common.blobstore.BlobContainer; //导入依赖的package包/类
@Override
public void verify(String seed) {
    BlobContainer testBlobContainer = blobStore.blobContainer(basePath.add(testBlobPrefix(seed)));
    DiscoveryNode localNode = clusterService.localNode();
    if (testBlobContainer.blobExists("master.dat")) {
        try  {
            testBlobContainer.writeBlob("data-" + localNode.getId() + ".dat", new BytesArray(seed));
        } catch (IOException exp) {
            throw new RepositoryVerificationException(repositoryName, "store location [" + blobStore + "] is not accessible on the node [" + localNode + "]", exp);
        }
    } else {
        throw new RepositoryVerificationException(repositoryName, "a file written by master to the store [" + blobStore + "] cannot be accessed on the node [" + localNode + "]. "
                + "This might indicate that the store [" + blobStore + "] is not shared between this node and the master node or "
                + "that permissions on the store don't allow reading files written by the master node");
    }
}