Java AmazonS3类代码示例

import com.amazonaws.services.s3.AmazonS3; //导入依赖的package包/类
@Autowired
public StorageService(AmazonS3 s3Client) throws IOException {
    isLocalFileSystem = s3Client == null;

    if (isLocalFileSystem) {
        this.s3Client = null;
        this.buckets = null;
        localFileSystemDirectory =
                new File(String.format("%s/%s", new File("build").getAbsolutePath(), "storage"));
        FileUtils.forceMkdir(localFileSystemDirectory);
    } else {
        this.s3Client = s3Client;
        this.buckets = new ConcurrentHashMap<>();
        this.s3Client.listBuckets().forEach(bucket -> buckets.put(bucket.getName(), bucket));
    }
}
 

import com.amazonaws.services.s3.AmazonS3; //导入依赖的package包/类
@Test
void testExecutionExceptionOnClose() throws Exception {
    // test failed close
    final AmazonS3 mockedS3 = mock(AmazonS3.class);
    final S3WritableObjectChannelBuilder builder = defaultBuilder("id")
            .closeExecutorOnChannelClose(false)
            .amazonS3(mockedS3);
    s3channel = (S3AppendableObjectChannel) builder.build();
    when(mockedS3.completeMultipartUpload(any())).thenThrow(new TestException());

    assertThrows(TestException.class, () -> s3channel.close());

    s3channel.close(); // already closed, no effect, should we throw exception?

    while (s3channel.getCancellation() == null) {
        Thread.sleep(10);
    }
    s3channel.getCancellation().get();

    assertTrue(!s3channel.getCancellation().isCompletedExceptionally());
    verify(mockedS3, times(1)).abortMultipartUpload(any());
}
 

import com.amazonaws.services.s3.AmazonS3; //导入依赖的package包/类
public static String webHookDump(InputStream stream, String school, String extension) {
    if (stream != null) {
        extension = extension == null || extension.isEmpty() ? ".xml" : extension.contains(".") ? extension : "." + extension;
        String fileName = "webhooks/" + school + "/" + school + "_" + Clock.getCurrentDateDashes() + "_" + Clock.getCurrentTime() + extension;
        AmazonS3 s3 = new AmazonS3Client();
        Region region = Region.getRegion(Regions.US_WEST_2);
        s3.setRegion(region);
        try {
            File file = CustomUtilities.inputStreamToFile(stream);
            s3.putObject(new PutObjectRequest(name, fileName, file));
            return CustomUtilities.fileToString(file);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    return "";
}
 

import com.amazonaws.services.s3.AmazonS3; //导入依赖的package包/类
@Test
public void checkBucketIndex() throws Exception {
  testWithAUser(
      (v) -> {
        String userId = v.getUserId();
        AmazonS3 s3 =
            createS3(
                v.getS3Credentials().get(0).getAccessKey(),
                v.getS3Credentials().get(0).getSecretKey());
        String bucketName = userId.toLowerCase();

        // not exist
        RGW_ADMIN.checkBucketIndex(bucketName, true, true);

        s3.createBucket(bucketName);
        // Do not know how to check the behavior...
        Optional result = RGW_ADMIN.checkBucketIndex(bucketName, true, true);
        assertTrue(result.isPresent());
      });
}
 

import com.amazonaws.services.s3.AmazonS3; //导入依赖的package包/类
@Test
void testFailedUploadPart() throws Exception {
    final AmazonS3 mocked = mock(AmazonS3.class);
    s3channel = (S3AppendableObjectChannel) defaultBuilder("id")
            .failedPartUploadRetries(3)
            .amazonS3(mocked)
            .build();
    when(mocked.uploadPart(any())).thenThrow(new TestException());

    s3channel.skip(MIN_PART_SIZE).write(ByteBuffer.allocate(123));
    while (s3channel.getCancellation() == null) {
        Thread.sleep(25);
    }
    s3channel.getCancellation().get();
    assertTrue(!s3channel.getCancellation().isCompletedExceptionally());
    assertFalse(s3channel.isOpen());

    //coverage
    s3channel.startWorker(new UploadPartRequest().withPartNumber(1), 0);

    assertThrows(IllegalStateException.class, () -> s3channel.write(ByteBuffer.allocate(1)));

    verify(mocked, times(1)).abortMultipartUpload(any());
}
 

import com.amazonaws.services.s3.AmazonS3; //导入依赖的package包/类
@Test
public void copyCheckTransferManagerIsShutdown() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(mockedTransferManager.copy(any(CopyObjectRequest.class), any(AmazonS3.class),
      any(TransferStateChangeListener.class))).thenReturn(copy);
  TransferProgress transferProgress = new TransferProgress();
  when(copy.getProgress()).thenReturn(transferProgress);
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  s3s3Copier.copy();
  verify(mockedTransferManager).shutdownNow();
}
 

import com.amazonaws.services.s3.AmazonS3; //导入依赖的package包/类
@Test
public void copySafelyShutDownTransferManagerWhenNotInitialised() throws Exception {
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();
  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenThrow(new RuntimeException("error in instance"));
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  try {
    s3s3Copier.copy();
  } catch (RuntimeException e) {
    assertThat(e.getMessage(), is("error in instance"));
  }
}
 

import com.amazonaws.services.s3.AmazonS3; //导入依赖的package包/类
@Test
public void copyCheckTransferManagerIsShutdownWhenSubmittingJobExceptionsAreThrown() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  when(mockedTransferManager.copy(any(CopyObjectRequest.class), any(AmazonS3.class),
      any(TransferStateChangeListener.class))).thenThrow(new AmazonServiceException("MyCause"));
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  try {
    s3s3Copier.copy();
    fail("exception should have been thrown");
  } catch (CircusTrainException e) {
    verify(mockedTransferManager).shutdownNow();
    assertThat(e.getCause().getMessage(), startsWith("MyCause"));
  }
}
 

import com.amazonaws.services.s3.AmazonS3; //导入依赖的package包/类
@Test
public void copyCheckTransferManagerIsShutdownWhenCopyExceptionsAreThrown() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(copy.getProgress()).thenReturn(new TransferProgress());
  when(mockedTransferManager.copy(any(CopyObjectRequest.class), any(AmazonS3.class),
      any(TransferStateChangeListener.class))).thenReturn(copy);
  doThrow(new AmazonClientException("cause")).when(copy).waitForCompletion();
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  try {
    s3s3Copier.copy();
    fail("exception should have been thrown");
  } catch (CircusTrainException e) {
    verify(mockedTransferManager).shutdownNow();
    assertThat(e.getCause().getMessage(), is("cause"));
  }
}
 

import com.amazonaws.services.s3.AmazonS3; //导入依赖的package包/类
@NotNull private SamInputResource getS3SamInputResource(BamFile bamFile) {
    final Bucket bucket = bucketManager.loadBucket(bamFile.getBucketId());
    Assert.notNull(bucket, getMessage(MessagesConstants.ERROR_S3_BUCKET));
    final AmazonS3 s3Client = new AmazonS3Client(new BasicAWSCredentials(bucket.getAccessKeyId(),
            bucket.getSecretAccessKey()));
    return SamInputResource.of(s3Client.generatePresignedUrl(bucket.getBucketName(), bamFile.getPath(),
            Utils.getTimeForS3URL()));
}
 

import com.amazonaws.services.s3.AmazonS3; //导入依赖的package包/类
public Object get(Object[] params) {
    AmazonS3 s3Client = new AmazonS3Client(new ProfileCredentialsProvider());
    try {
        S3Object object = s3Client.getObject(new GetObjectRequest(bucketName, params[0].toString()));
        InputStream objectData = object.getObjectContent();
        byte[] bytes = IOUtils.toByteArray(is);
        ByteBuffer b = ByteBuffer.wrap(bytes);
        return b;
    } catch (AmazonServiceException ase) {
        System.out.println("Caught an AmazonServiceException, which " +
            "means your request made it " +
            "to Amazon S3, but was rejected with an error response" +
            " for some reason.");
        System.out.println("Error Message:" + ase.getMessage());
        System.out.println("HTTP Status Code:" + ase.getStatusCode());
        System.out.println("AWS Error Code:" + ase.getErrorCode());
        System.out.println("Error Type:" + ase.getErrorType());
        System.out.println("Request ID:" + ase.getRequestId());
    } catch (AmazonClientException ace) {
        System.out.println("Caught an AmazonClientException, which " +
            "means the client encountered " +
            "an internal error while trying to " +
            "communicate with S3, " +
            "such as not being able to access the network.");
        System.out.println("Error Message: " + ace.getMessage());
    }

}
 

import com.amazonaws.services.s3.AmazonS3; //导入依赖的package包/类
/**
 * Creates the S3 client {@link Bean}.
 *
 * Uses the default client, but if a region is unspecified, uses {@code us-east-1}.
 *
 * @return The S3 client.
 */
@Bean
public AmazonS3 s3client() {
	try {
		return AmazonS3ClientBuilder.defaultClient();
	} catch (SdkClientException exception) {
		API_LOG.info("Default S3 client failed to build, trying again with region us-east-1", exception);
		return planB();
	}
}
 

import com.amazonaws.services.s3.AmazonS3; //导入依赖的package包/类
@Override
public Map<String, BlobMetaData> listBlobsByPrefix(@Nullable String blobNamePrefix) throws IOException {
    return AccessController.doPrivileged((PrivilegedAction<Map<String, BlobMetaData>>) () -> {
        MapBuilder<String, BlobMetaData> blobsBuilder = MapBuilder.newMapBuilder();
        AmazonS3 client = blobStore.client();
        SocketAccess.doPrivilegedVoid(() -> {
            ObjectListing prevListing = null;
            while (true) {
                ObjectListing list;
                if (prevListing != null) {
                    list = client.listNextBatchOfObjects(prevListing);
                } else {
                    if (blobNamePrefix != null) {
                        list = client.listObjects(blobStore.bucket(), buildKey(blobNamePrefix));
                    } else {
                        list = client.listObjects(blobStore.bucket(), keyPath);
                    }
                }
                for (S3ObjectSummary summary : list.getObjectSummaries()) {
                    String name = summary.getKey().substring(keyPath.length());
                    blobsBuilder.put(name, new PlainBlobMetaData(name, summary.getSize()));
                }
                if (list.isTruncated()) {
                    prevListing = list;
                } else {
                    break;
                }
            }
        });
        return blobsBuilder.immutableMap();
    });
}
 

import com.amazonaws.services.s3.AmazonS3; //导入依赖的package包/类
public CompleteMultipartUpload(String uploadId, AmazonS3 s3,
        PutObjectRequest putObjectRequest, List<Future<PartETag>> futures,
        List<PartETag> eTagsBeforeResume, ProgressListenerChain progressListenerChain,
        UploadMonitor monitor) {
    this.uploadId = uploadId;
    this.s3 = s3;
    this.origReq = putObjectRequest;
    this.futures = futures;
    this.eTagsBeforeResume = eTagsBeforeResume;
    this.listener = progressListenerChain;
    this.monitor = monitor;
}
 

import com.amazonaws.services.s3.AmazonS3; //导入依赖的package包/类
private AmazonS3 cachedWrapper(AmazonS3 client) {
    TestAmazonS3 wrapper = clients.get(client);
    if (wrapper == null) {
        wrapper = new TestAmazonS3(client, settings);
        clients.put(client, wrapper);
    }
    return wrapper;
}