Java TransferManager类代码示例

import com.amazonaws.services.s3.transfer.TransferManager; //导入依赖的package包/类
public static void uploadDir(String dir_path, String bucket_name,
        String key_prefix, boolean recursive, boolean pause)
{
    System.out.println("directory: " + dir_path + (recursive ?
                " (recursive)" : "") + (pause ? " (pause)" : ""));

    TransferManager xfer_mgr = new TransferManager();
    try {
        MultipleFileUpload xfer = xfer_mgr.uploadDirectory(bucket_name,
                key_prefix, new File(dir_path), recursive);
        // loop with Transfer.isDone()
        XferMgrProgress.showTransferProgress(xfer);
        // or block with Transfer.waitForCompletion()
        XferMgrProgress.waitForCompletion(xfer);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    xfer_mgr.shutdownNow();
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入依赖的package包/类
@Override
public String retrieve(String file) throws Exception {
	LogUtils.debug(LOG_TAG, "Downloading file: " + file);
       TransferManager tm = new TransferManager(new DefaultAWSCredentialsProviderChain());
       // TransferManager processes all transfers asynchronously, 
       // so this call will return immediately.
       File downloadedFile = new File(Constants.MCSFS_WORKING_DIR + Constants.S3_WORKING_DIR + file + System.currentTimeMillis());
	downloadedFile.getParentFile().mkdirs();
	downloadedFile.createNewFile();
	Download download = tm.download(bucketName, file, downloadedFile);
       download.waitForCompletion();
       LogUtils.debug(LOG_TAG, "Successfully downloaded file from bucket.\nName: " + file + "\nBucket name: " +
               bucketName);
       tm.shutdownNow();
	return downloadedFile.getAbsolutePath();
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入依赖的package包/类
public static void uploadDirWithSubprogress(String dir_path,
        String bucket_name, String key_prefix, boolean recursive,
        boolean pause)
{
    System.out.println("directory: " + dir_path + (recursive ?
                " (recursive)" : "") + (pause ? " (pause)" : ""));

    TransferManager xfer_mgr = new TransferManager();
    try {
        MultipleFileUpload multi_upload = xfer_mgr.uploadDirectory(
                bucket_name, key_prefix, new File(dir_path), recursive);
        // loop with Transfer.isDone()
        XferMgrProgress.showMultiUploadProgress(multi_upload);
        // or block with Transfer.waitForCompletion()
        XferMgrProgress.waitForCompletion(multi_upload);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    xfer_mgr.shutdownNow();
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入依赖的package包/类
private String uploadToS3(String bucket, String key, MultipartFile file) {
    final AmazonS3 s3 = s3ClientFactory.createClient();
    final TransferManager transferManager = TransferManagerBuilder.standard().withS3Client(s3).build();
    try {
        ObjectMetadata metadata = new ObjectMetadata();
        metadata.setContentLength(file.getSize());
        metadata.setContentType(file.getContentType());

        byte[] resultByte = DigestUtils.md5(file.getBytes());
        String streamMD5 = new String(Base64.encodeBase64(resultByte));
        metadata.setContentMD5(streamMD5);

        Upload upload = transferManager.upload(bucket, key, file.getInputStream(), metadata);
        upload.waitForCompletion();
        return streamMD5;
    } catch (AmazonServiceException | InterruptedException | IOException e) {
        logger.error("Error uploading file: {}", e.toString());
        return null;
    } finally {
        transferManager.shutdownNow();
    }
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入依赖的package包/类
/**
 * Gets an Amazon transfer manager.
 *
 * @return a transfer manager
 */
public TransferManager getTransferManager() {
    if (transferManager == null) {
        transferManager = TransferManagerBuilder.standard().withS3Client(getAmazonS3Client()).build();
    }
    return transferManager;
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入依赖的package包/类
@Test
public void copyCheckTransferManagerIsShutdown() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(mockedTransferManager.copy(any(CopyObjectRequest.class), any(AmazonS3.class),
      any(TransferStateChangeListener.class))).thenReturn(copy);
  TransferProgress transferProgress = new TransferProgress();
  when(copy.getProgress()).thenReturn(transferProgress);
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  s3s3Copier.copy();
  verify(mockedTransferManager).shutdownNow();
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入依赖的package包/类
@Test
public void copyCheckTransferManagerIsShutdownWhenSubmittingJobExceptionsAreThrown() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  when(mockedTransferManager.copy(any(CopyObjectRequest.class), any(AmazonS3.class),
      any(TransferStateChangeListener.class))).thenThrow(new AmazonServiceException("MyCause"));
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  try {
    s3s3Copier.copy();
    fail("exception should have been thrown");
  } catch (CircusTrainException e) {
    verify(mockedTransferManager).shutdownNow();
    assertThat(e.getCause().getMessage(), startsWith("MyCause"));
  }
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入依赖的package包/类
@Test
public void copyCheckTransferManagerIsShutdownWhenCopyExceptionsAreThrown() throws Exception {
  client.putObject("source", "data", inputData);
  Path sourceBaseLocation = new Path("s3://source/");
  Path replicaLocation = new Path("s3://target/");
  List<Path> sourceSubLocations = new ArrayList<>();

  TransferManagerFactory mockedTransferManagerFactory = Mockito.mock(TransferManagerFactory.class);
  TransferManager mockedTransferManager = Mockito.mock(TransferManager.class);
  when(mockedTransferManagerFactory.newInstance(any(AmazonS3.class), eq(s3S3CopierOptions)))
      .thenReturn(mockedTransferManager);
  Copy copy = Mockito.mock(Copy.class);
  when(copy.getProgress()).thenReturn(new TransferProgress());
  when(mockedTransferManager.copy(any(CopyObjectRequest.class), any(AmazonS3.class),
      any(TransferStateChangeListener.class))).thenReturn(copy);
  doThrow(new AmazonClientException("cause")).when(copy).waitForCompletion();
  S3S3Copier s3s3Copier = new S3S3Copier(sourceBaseLocation, sourceSubLocations, replicaLocation, s3ClientFactory,
      mockedTransferManagerFactory, listObjectsRequestFactory, registry, s3S3CopierOptions);
  try {
    s3s3Copier.copy();
    fail("exception should have been thrown");
  } catch (CircusTrainException e) {
    verify(mockedTransferManager).shutdownNow();
    assertThat(e.getCause().getMessage(), is("cause"));
  }
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入依赖的package包/类
@Test
public void testS3Persister() throws Exception {
    Upload upload = mock(Upload.class);
    when(upload.waitForUploadResult()).thenReturn(new UploadResult());
    TransferManager transferManager = mock(TransferManager.class);
    when(transferManager.upload(anyString(), anyString(), any())).thenReturn(upload);

    S3Persister s3Persister = new S3Persister(transferManager, "foo");
    s3Persister.saveMetrics("foo", "bar");

    verify(transferManager, times(1)).upload(anyString(), anyString(), any());
    verify(transferManager, times(1)).shutdownNow();
    verifyNoMoreInteractions(transferManager);
    verify(upload, times(1)).waitForCompletion();
    verifyNoMoreInteractions(upload);
    assertFalse(new File("foo").exists());
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入依赖的package包/类
@Override
@Guarded(by = STARTED)
public Blob create(final InputStream blobData, final Map<String, String> headers) {
  checkNotNull(blobData);

  return create(headers, destination -> {
      try (InputStream data = blobData) {
        MetricsInputStream input = new MetricsInputStream(data);
        TransferManager transferManager = new TransferManager(s3);
        transferManager.upload(getConfiguredBucket(), destination, input, new ObjectMetadata())
            .waitForCompletion();
        return input.getMetrics();
      } catch (InterruptedException e) {
        throw new BlobStoreException("error uploading blob", e, null);
      }
    });
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入依赖的package包/类
/**
 * Constructs a new watcher for copy operation, and then immediately submits
 * it to the thread pool.
 *
 * @param manager
 *            The {@link TransferManager} that owns this copy request.
 * @param threadPool
 *            The {@link ExecutorService} to which we should submit new
 *            tasks.
 * @param multipartCopyCallable
 *            The callable responsible for processing the copy
 *            asynchronously
 * @param copyObjectRequest
 *            The original CopyObject request
 */
public static CopyMonitor create(
        TransferManager manager,
        CopyImpl transfer,
        ExecutorService threadPool,
        CopyCallable multipartCopyCallable,
        CopyObjectRequest copyObjectRequest,
        ProgressListenerChain progressListenerChain) {

    CopyMonitor copyMonitor = new CopyMonitor(manager, transfer,
            threadPool, multipartCopyCallable, copyObjectRequest,
            progressListenerChain);
    Future<CopyResult> thisFuture = threadPool.submit(copyMonitor);
    // Use an atomic compareAndSet to prevent a possible race between the
    // setting of the CopyMonitor's futureReference, and setting the
    // CompleteMultipartCopy's futureReference within the call() method.
    // We only want to set the futureReference to CopyMonitor's futureReference if the
    // current value is null, otherwise the futureReference that's set is
    // CompleteMultipartCopy's which is ultimately what we want.
    copyMonitor.futureReference.compareAndSet(null, thisFuture);
    return copyMonitor;
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入依赖的package包/类
/**
 * Constructs a new upload watcher and then immediately submits it to
 * the thread pool.
 *
 * @param manager
 *            The {@link TransferManager} that owns this upload.
 * @param transfer
 *            The transfer being processed.
 * @param threadPool
 *            The {@link ExecutorService} to which we should submit new
 *            tasks.
 * @param multipartUploadCallable
 *            The callable responsible for processing the upload
 *            asynchronously
 * @param putObjectRequest
 *            The original putObject request
 * @param progressListenerChain
 *            A chain of listeners that wish to be notified of upload
 *            progress
 */
public static UploadMonitor create(
        TransferManager manager,
        UploadImpl transfer,
        ExecutorService threadPool,
        UploadCallable multipartUploadCallable,
        PutObjectRequest putObjectRequest,
        ProgressListenerChain progressListenerChain) {

    UploadMonitor uploadMonitor = new UploadMonitor(manager, transfer,
            threadPool, multipartUploadCallable, putObjectRequest,
            progressListenerChain);
    Future<UploadResult> thisFuture = threadPool.submit(uploadMonitor);
    // Use an atomic compareAndSet to prevent a possible race between the
    // setting of the UploadMonitor's futureReference, and setting the
    // CompleteMultipartUpload's futureReference within the call() method.
    // We only want to set the futureReference to UploadMonitor's futureReference if the
    // current value is null, otherwise the futureReference that's set is
    // CompleteMultipartUpload's which is ultimately what we want.
    uploadMonitor.futureReference.compareAndSet(null, thisFuture);
    return uploadMonitor;
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入依赖的package包/类
/**
 * Tests if an object can be uploaded asynchronously
 *
 * @throws Exception not expected
 */
@Test
public void shouldUploadInParallel() throws Exception {
  final File uploadFile = new File(UPLOAD_FILE_NAME);

  s3Client.createBucket(BUCKET_NAME);

  final TransferManager transferManager = createDefaultTransferManager();
  final Upload upload =
      transferManager.upload(new PutObjectRequest(BUCKET_NAME, UPLOAD_FILE_NAME, uploadFile));
  final UploadResult uploadResult = upload.waitForUploadResult();

  assertThat(uploadResult.getKey(), equalTo(UPLOAD_FILE_NAME));

  final S3Object getResult = s3Client.getObject(BUCKET_NAME, UPLOAD_FILE_NAME);
  assertThat(getResult.getKey(), equalTo(UPLOAD_FILE_NAME));
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入依赖的package包/类
/**
 * Verify that range-downloads work.
 *
 * @throws Exception not expected
 */
@Test
public void checkRangeDownloads() throws Exception {
  final File uploadFile = new File(UPLOAD_FILE_NAME);

  s3Client.createBucket(BUCKET_NAME);

  final TransferManager transferManager = createDefaultTransferManager();
  final Upload upload =
      transferManager.upload(new PutObjectRequest(BUCKET_NAME, UPLOAD_FILE_NAME, uploadFile));
  upload.waitForUploadResult();

  final File downloadFile = File.createTempFile(UUID.randomUUID().toString(), null);
  transferManager
      .download(new GetObjectRequest(BUCKET_NAME, UPLOAD_FILE_NAME).withRange(1, 2),
          downloadFile)
      .waitForCompletion();
  assertThat("Invalid file length", downloadFile.length(), is(2L));

  transferManager
      .download(new GetObjectRequest(BUCKET_NAME, UPLOAD_FILE_NAME).withRange(0, 1000),
          downloadFile)
      .waitForCompletion();
  assertThat("Invalid file length", downloadFile.length(), is(uploadFile.length()));
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入依赖的package包/类
private TransferManager createTransferManager(final long multipartUploadThreshold,
    final long multipartUploadPartSize,
    final long multipartCopyThreshold,
    final long multipartCopyPartSize) {
  final ThreadFactory threadFactory = new ThreadFactory() {
    private int threadCount = 1;

    @Override
    public Thread newThread(final Runnable r) {
      final Thread thread = new Thread(r);
      thread.setName("s3-transfer-" + threadCount++);

      return thread;
    }
  };

  return TransferManagerBuilder.standard()
      .withS3Client(s3Client)
      .withExecutorFactory(() -> Executors.newFixedThreadPool(THREAD_COUNT, threadFactory))
      .withMultipartUploadThreshold(multipartUploadThreshold)
      .withMinimumUploadPartSize(multipartUploadPartSize)
      .withMultipartCopyPartSize(multipartCopyPartSize)
      .withMultipartCopyThreshold(multipartCopyThreshold)
      .build();
}