Java TransferManager.shutdownNow方法代码示例

import com.amazonaws.services.s3.transfer.TransferManager; //导入方法依赖的package包/类
public static void uploadDir(String dir_path, String bucket_name,
        String key_prefix, boolean recursive, boolean pause)
{
    System.out.println("directory: " + dir_path + (recursive ?
                " (recursive)" : "") + (pause ? " (pause)" : ""));

    TransferManager xfer_mgr = new TransferManager();
    try {
        MultipleFileUpload xfer = xfer_mgr.uploadDirectory(bucket_name,
                key_prefix, new File(dir_path), recursive);
        // loop with Transfer.isDone()
        XferMgrProgress.showTransferProgress(xfer);
        // or block with Transfer.waitForCompletion()
        XferMgrProgress.waitForCompletion(xfer);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    xfer_mgr.shutdownNow();
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入方法依赖的package包/类
@Override
public String retrieve(String file) throws Exception {
	LogUtils.debug(LOG_TAG, "Downloading file: " + file);
       TransferManager tm = new TransferManager(new DefaultAWSCredentialsProviderChain());
       // TransferManager processes all transfers asynchronously, 
       // so this call will return immediately.
       File downloadedFile = new File(Constants.MCSFS_WORKING_DIR + Constants.S3_WORKING_DIR + file + System.currentTimeMillis());
	downloadedFile.getParentFile().mkdirs();
	downloadedFile.createNewFile();
	Download download = tm.download(bucketName, file, downloadedFile);
       download.waitForCompletion();
       LogUtils.debug(LOG_TAG, "Successfully downloaded file from bucket.\nName: " + file + "\nBucket name: " +
               bucketName);
       tm.shutdownNow();
	return downloadedFile.getAbsolutePath();
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入方法依赖的package包/类
public static void uploadDirWithSubprogress(String dir_path,
        String bucket_name, String key_prefix, boolean recursive,
        boolean pause)
{
    System.out.println("directory: " + dir_path + (recursive ?
                " (recursive)" : "") + (pause ? " (pause)" : ""));

    TransferManager xfer_mgr = new TransferManager();
    try {
        MultipleFileUpload multi_upload = xfer_mgr.uploadDirectory(
                bucket_name, key_prefix, new File(dir_path), recursive);
        // loop with Transfer.isDone()
        XferMgrProgress.showMultiUploadProgress(multi_upload);
        // or block with Transfer.waitForCompletion()
        XferMgrProgress.waitForCompletion(multi_upload);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    xfer_mgr.shutdownNow();
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入方法依赖的package包/类
private String uploadToS3(String bucket, String key, MultipartFile file) {
    final AmazonS3 s3 = s3ClientFactory.createClient();
    final TransferManager transferManager = TransferManagerBuilder.standard().withS3Client(s3).build();
    try {
        ObjectMetadata metadata = new ObjectMetadata();
        metadata.setContentLength(file.getSize());
        metadata.setContentType(file.getContentType());

        byte[] resultByte = DigestUtils.md5(file.getBytes());
        String streamMD5 = new String(Base64.encodeBase64(resultByte));
        metadata.setContentMD5(streamMD5);

        Upload upload = transferManager.upload(bucket, key, file.getInputStream(), metadata);
        upload.waitForCompletion();
        return streamMD5;
    } catch (AmazonServiceException | InterruptedException | IOException e) {
        logger.error("Error uploading file: {}", e.toString());
        return null;
    } finally {
        transferManager.shutdownNow();
    }
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入方法依赖的package包/类
public static void downloadDir(String bucket_name, String key_prefix,
      String dir_path, boolean pause)
{
    System.out.println("downloading to directory: " + dir_path +
          (pause ?  " (pause)" : ""));

    TransferManager xfer_mgr = new TransferManager();
    try {
        MultipleFileDownload xfer = xfer_mgr.downloadDirectory(
                bucket_name, key_prefix, new File(dir_path));
        // loop with Transfer.isDone()
        XferMgrProgress.showTransferProgress(xfer);
        // or block with Transfer.waitForCompletion()
        XferMgrProgress.waitForCompletion(xfer);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    xfer_mgr.shutdownNow();
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入方法依赖的package包/类
public static void downloadFile(String bucket_name, String key_name,
      String file_path, boolean pause)
{
    System.out.println("Downloading to file: " + file_path +
          (pause ? " (pause)" : ""));

    File f = new File(file_path);
    TransferManager xfer_mgr = new TransferManager();
    try {
        Download xfer = xfer_mgr.download(bucket_name, key_name, f);
        // loop with Transfer.isDone()
        XferMgrProgress.showTransferProgress(xfer);
        // or block with Transfer.waitForCompletion()
        XferMgrProgress.waitForCompletion(xfer);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    xfer_mgr.shutdownNow();
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入方法依赖的package包/类
public static void copyObjectSimple(String from_bucket, String from_key,
        String to_bucket, String to_key) {
    System.out.println("Copying s3 object: " + from_key);
    System.out.println("      from bucket: " + from_bucket);
    System.out.println("     to s3 object: " + to_bucket);
    System.out.println("        in bucket: " + to_key);

    TransferManager xfer_mgr = new TransferManager();
    try {
        Copy xfer = xfer_mgr.copy(from_bucket, from_key, to_bucket, to_key);
        // loop with Transfer.isDone()
        XferMgrProgress.showTransferProgress(xfer);
        // or block with Transfer.waitForCompletion()
        XferMgrProgress.waitForCompletion(xfer);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    xfer_mgr.shutdownNow();
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入方法依赖的package包/类
public static void uploadFileList(String[] file_paths, String bucket_name,
        String key_prefix, boolean pause)
{
    System.out.println("file list: " + Arrays.toString(file_paths) +
            (pause ? " (pause)" : ""));
    // convert the file paths to a list of File objects (required by the
    // uploadFileList method)
    ArrayList<File> files = new ArrayList<File>();
    for (String path : file_paths) {
        files.add(new File(path));
    }

    TransferManager xfer_mgr = new TransferManager();
    try {
        MultipleFileUpload xfer = xfer_mgr.uploadFileList(bucket_name,
                key_prefix, new File("."), files);
        // loop with Transfer.isDone()
        XferMgrProgress.showTransferProgress(xfer);
        // or block with Transfer.waitForCompletion()
        XferMgrProgress.waitForCompletion(xfer);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    xfer_mgr.shutdownNow();
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入方法依赖的package包/类
@Override
public Integer call() throws Exception {
    TransferManager t = new TransferManager(amazonS3Client);

    ObjectMetadata objectMetadata = new ObjectMetadata();
    objectMetadata.setUserMetadata(metadata);
    if(sse) {
        objectMetadata.setSSEAlgorithm(SSEAlgorithm.AES256.getAlgorithm());
    }

    Upload u = t.upload(new PutObjectRequest(bucket, key, inputFile).withMetadata(objectMetadata));

    // TODO this listener spews out garbage >100% on a retry, add a test to verify
    if (progressListener != null) {
        progressListener.withTransferProgress(new TransferProgressWrapper(u.getProgress()));
        u.addProgressListener(progressListener);
    }
    try {
        u.waitForCompletion();
    } finally {
        t.shutdownNow();
    }
    return 0;
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入方法依赖的package包/类
@Override
public void store(File file) throws Exception {
       
	LogUtils.debug(LOG_TAG, "Uploading new file. Name: " + file.getName());
       TransferManager tm = new TransferManager(new DefaultAWSCredentialsProviderChain());
       // TransferManager processes all transfers asynchronously, 
       // so this call will return immediately.
       Upload upload = tm.upload(bucketName, file.getName(), file);
       upload.waitForCompletion();
       LogUtils.debug(LOG_TAG, "Successfully uploaded file to bucket.\nName: " + file.getName() + "\nBucket name: " +
               bucketName);
       tm.shutdownNow();
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
    // create the AWS S3 Client
    AmazonS3 s3 = AWSS3Factory.getS3Client();

    // retrieve the key value from user
    System.out.println( "Enter the object key:" );
    String key = new BufferedReader( new InputStreamReader( System.in ) ).readLine();

    // print start time
    Date start_date = new Date();
    System.out.println(start_date.toString());

    // file will be placed in temp dir with .tmp extension
    File file = File.createTempFile("read-large-object-tm", null);

    TransferManager tm = TransferManagerBuilder.standard()
            .withS3Client(s3)
            .build();

    // download the object to file
    Download download = tm.download(AWSS3Factory.S3_BUCKET, key, file);

    // block until download finished
    download.waitForCompletion();

    tm.shutdownNow();

    // print end time
    Date end_date = new Date();
    System.out.println(end_date.toString());
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入方法依赖的package包/类
public static void uploadFile(String file_path, String bucket_name,
        String key_prefix, boolean pause)
{
    System.out.println("file: " + file_path +
            (pause ? " (pause)" : ""));

    String key_name = null;
    if (key_prefix != null) {
        key_name = key_prefix + '/' + file_path;
    } else {
        key_name = file_path;
    }

    File f = new File(file_path);
    TransferManager xfer_mgr = new TransferManager();
    try {
        Upload xfer = xfer_mgr.upload(bucket_name, key_name, f);
        // loop with Transfer.isDone()
        XferMgrProgress.showTransferProgress(xfer);
        //  or block with Transfer.waitForCompletion()
        XferMgrProgress.waitForCompletion(xfer);
    } catch (AmazonServiceException e) {
        System.err.println(e.getErrorMessage());
        System.exit(1);
    }
    xfer_mgr.shutdownNow();
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入方法依赖的package包/类
private void cleanupMultiPartUploads( AmazonS3 s3Client, String bucket ){
	TransferManager tm = new TransferManager(s3Client);        
   try {
   	tm.abortMultipartUploads(bucket, new Date(System.currentTimeMillis() - DateUtil.DAY_MS ));
   } catch (AmazonClientException amazonClientException) {
   	cfEngine.log("AmazonS3Write.BackgroundUploader.cleanupMultiPartUploads():" + amazonClientException.getMessage() );
   }
   tm.shutdownNow(true);
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入方法依赖的package包/类
private void waitForUpload(Transfer upload, TransferManager tm) throws AmazonServiceException,
    AmazonClientException, InterruptedException {
  long bytes = upload.getProgress().getTotalBytesToTransfer();

  console.write(new Size(bytes) + " to upload"); //$NON-NLS-1$

  long fraction = bytes / 50;

  upload.addProgressListener(new SharpProgressListener(fraction));
  upload.waitForCompletion();

  tm.shutdownNow();
  console.write("\nDone"); //$NON-NLS-1$
}
 

import com.amazonaws.services.s3.transfer.TransferManager; //导入方法依赖的package包/类
@Override
public void upload(BackupRestoreContext ctx) throws Exception {
    final String localLocation = ctx.getLocalLocation();
    final String backupName = ctx.getName();
    final String nodeId = ctx.getNodeId();
    final String key = getPrefixKey(ctx) + "/" + nodeId;
    LOGGER.info("Backup key: " + key);
    final TransferManager tx = getS3TransferManager(ctx);
    final File dataDirectory = new File(localLocation);

    try {
        // Ex: data/<keyspace>/<cf>/snapshots/</snapshot-dir>/<files>
        for (File keyspaceDir : dataDirectory.listFiles()) {
            if (keyspaceDir.isFile()) {
                // Skip any files in the data directory.
                // Only enter keyspace directory.
                continue;
            }
            LOGGER.info("Entering keyspace: {}", keyspaceDir.getName());
            for (File cfDir : getColumnFamilyDir(keyspaceDir)) {
                LOGGER.info("Entering column family dir: {}", cfDir.getName());
                File snapshotDir = new File(cfDir, "snapshots");
                File backupDir = new File(snapshotDir, backupName);
                if (!StorageUtil.isValidBackupDir(keyspaceDir, cfDir, snapshotDir, backupDir)) {
                    LOGGER.info("Skipping directory: {}",
                            snapshotDir.getAbsolutePath());
                    continue;
                }
                LOGGER.info(
                        "Valid backup directories. KeyspaceDir: {} | ColumnFamilyDir: {} | SnapshotDir: {} | BackupName: {}",
                        keyspaceDir.getAbsolutePath(), cfDir.getAbsolutePath(),
                        snapshotDir.getAbsolutePath(), backupName);

                final Optional<File> snapshotDirectory = StorageUtil.getValidSnapshotDirectory(
                        snapshotDir, backupName);
                LOGGER.info("Valid snapshot directory: {}",
                        snapshotDirectory.isPresent());
                if (snapshotDirectory.isPresent()) {
                    // Upload this directory
                    LOGGER.info("Going to upload directory: {}",
                            snapshotDirectory.get().getAbsolutePath());

                    uploadDirectory(
                            tx,
                            getBucketName(ctx),
                            key,
                            keyspaceDir.getName(),
                            cfDir.getName(),
                            snapshotDirectory.get());
                } else {
                    LOGGER.warn(
                            "Snapshots directory: {} doesn't contain the current backup directory: {}",
                            snapshotDir.getName(), backupName);
                }
            }
        }
        LOGGER.info("Done uploading snapshots for backup: {}", backupName);
    } catch (Exception e) {
        LOGGER.info("Failed uploading snapshots for backup: {}, error: {}", backupName, e);
        throw new Exception(e);
    } finally {
        tx.shutdownNow();
    }
}