Java InitiateMultipartUploadResult.getUploadId方法代码示例

import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; //导入方法依赖的package包/类
/**
 * Tests if not yet completed / aborted multipart uploads are listed.
 *
 * @throws Exception not expected
 */
@Test
public void shouldListMultipartUploads() throws Exception {
  s3Client.createBucket(BUCKET_NAME);

  assertThat(s3Client.listMultipartUploads(new ListMultipartUploadsRequest(BUCKET_NAME))
      .getMultipartUploads(), is(empty()));

  final InitiateMultipartUploadResult initiateMultipartUploadResult = s3Client
      .initiateMultipartUpload(new InitiateMultipartUploadRequest(BUCKET_NAME, UPLOAD_FILE_NAME));
  final String uploadId = initiateMultipartUploadResult.getUploadId();

  final MultipartUploadListing listing =
      s3Client.listMultipartUploads(new ListMultipartUploadsRequest(BUCKET_NAME));
  assertThat(listing.getMultipartUploads(), is(not(empty())));
  assertThat(listing.getBucketName(), equalTo(BUCKET_NAME));
  assertThat(listing.getMultipartUploads(), hasSize(1));
  final MultipartUpload upload = listing.getMultipartUploads().get(0);
  assertThat(upload.getUploadId(), equalTo(uploadId));
  assertThat(upload.getKey(), equalTo(UPLOAD_FILE_NAME));
}
 

import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; //导入方法依赖的package包/类
/**
 * Tests if a multipart upload can be aborted.
 *
 * @throws Exception not expected
 */
@Test
public void shouldAbortMultipartUpload() throws Exception {
  s3Client.createBucket(BUCKET_NAME);

  assertThat(s3Client.listMultipartUploads(new ListMultipartUploadsRequest(BUCKET_NAME))
      .getMultipartUploads(), is(empty()));

  final InitiateMultipartUploadResult initiateMultipartUploadResult = s3Client
      .initiateMultipartUpload(new InitiateMultipartUploadRequest(BUCKET_NAME, UPLOAD_FILE_NAME));
  final String uploadId = initiateMultipartUploadResult.getUploadId();

  assertThat(s3Client.listMultipartUploads(new ListMultipartUploadsRequest(BUCKET_NAME))
      .getMultipartUploads(), is(not(empty())));

  s3Client.abortMultipartUpload(
      new AbortMultipartUploadRequest(BUCKET_NAME, UPLOAD_FILE_NAME, uploadId));

  assertThat(s3Client.listMultipartUploads(new ListMultipartUploadsRequest(BUCKET_NAME))
      .getMultipartUploads(), is(empty()));
}
 

import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; //导入方法依赖的package包/类
/**
 * For the input file, initiate the upload and emit the UploadFileMetadata through the fileMetadataOutput,
 * uploadMetadataOutput ports.
 * @param tuple given tuple
 */
protected void processTuple(AbstractFileSplitter.FileMetadata tuple)
{
  if (currentWindowId <= windowDataManager.getLargestCompletedWindow()) {
    return;
  }
  String keyName = getKeyName(tuple.getFilePath());
  String uploadId = "";
  if (tuple.getNumberOfBlocks() > 1) {
    InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, keyName);
    initRequest.setObjectMetadata(createObjectMetadata());
    InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest);
    uploadId = initResponse.getUploadId();
  }
  UploadFileMetadata uploadFileMetadata = new UploadFileMetadata(tuple, uploadId, keyName);
  fileMetadataOutput.emit(uploadFileMetadata);
  uploadMetadataOutput.emit(uploadFileMetadata);
  currentWindowRecoveryState.add(uploadFileMetadata);
}
 

import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; //导入方法依赖的package包/类
@Test
public void testAtomicMpuAbort() throws Exception {
    String key = "testAtomicMpuAbort";
    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, key, BYTE_SOURCE.openStream(),
            metadata);

    InitiateMultipartUploadRequest initRequest =
            new InitiateMultipartUploadRequest(containerName, key);
    InitiateMultipartUploadResult initResponse =
            client.initiateMultipartUpload(initRequest);
    String uploadId = initResponse.getUploadId();

    client.abortMultipartUpload(new AbortMultipartUploadRequest(
                containerName, key, uploadId));

    S3Object object = client.getObject(containerName, key);
    assertThat(object.getObjectMetadata().getContentLength()).isEqualTo(
            BYTE_SOURCE.size());
    try (InputStream actual = object.getObjectContent();
            InputStream expected = BYTE_SOURCE.openStream()) {
        assertThat(actual).hasContentEqualTo(expected);
    }
}
 

import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; //导入方法依赖的package包/类
@Test
public void testPartNumberMarker() throws Exception {
    String blobName = "foo";
    InitiateMultipartUploadResult result = client.initiateMultipartUpload(
            new InitiateMultipartUploadRequest(containerName, blobName));
    ListPartsRequest request = new ListPartsRequest(containerName,
            blobName, result.getUploadId());

    client.listParts(request.withPartNumberMarker(0));

    try {
        client.listParts(request.withPartNumberMarker(1));
        Fail.failBecauseExceptionWasNotThrown(AmazonS3Exception.class);
    } catch (AmazonS3Exception e) {
        assertThat(e.getErrorCode()).isEqualTo("NotImplemented");
    }
}
 

import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; //导入方法依赖的package包/类
private String initiateMultipartUpload() throws IOException {

		boolean operationSuccessful = false;
		final InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(this.bucket, this.object);
		if (this.useRRS) {
			request.setStorageClass(StorageClass.ReducedRedundancy);
		} else {
			request.setStorageClass(StorageClass.Standard);
		}

		try {

			final InitiateMultipartUploadResult result = this.s3Client.initiateMultipartUpload(request);
			operationSuccessful = true;
			return result.getUploadId();

		} catch (AmazonServiceException e) {
			throw new IOException(StringUtils.stringifyException(e));
		} finally {
			if (!operationSuccessful) {
				abortUpload();
			}
		}
	}
 

import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; //导入方法依赖的package包/类
public String initiateMultipartUpload(S3Upload upload, String originalFilename) {
    try {
        runtimeCredentialsProvider.setAwsCredentials(upload.getAwsCredentials());
        amazonS3.setRegion(Region.getRegion(Regions.fromName(upload.getRegion())));
        ObjectMetadata objectMetadata = new ObjectMetadata();
        objectMetadata.addUserMetadata("original-filename", originalFilename);
        InitiateMultipartUploadResult result = amazonS3.initiateMultipartUpload(new InitiateMultipartUploadRequest(upload.getBucket(), upload.getKey(), objectMetadata));
        return result.getUploadId();
    } catch (AmazonClientException e) {
        throw new OmakaseRuntimeException(e);
    }
}
 

import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; //导入方法依赖的package包/类
private void multipartUpload(String key, File file, ObjectMetadata objectMetadata, Optional<StorageClass> maybeStorageClass) throws Exception {
  List<PartETag> partETags = new ArrayList<>();
  InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, key, objectMetadata);
  if (maybeStorageClass.isPresent()) {
    initRequest.setStorageClass(maybeStorageClass.get());
  }
  InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest);

  long contentLength = file.length();
  long partSize = configuration.getUploadPartSize();

  try {
    long filePosition = 0;
    for (int i = 1; filePosition < contentLength; i++) {
      partSize = Math.min(partSize, (contentLength - filePosition));
      UploadPartRequest uploadRequest = new UploadPartRequest()
          .withBucketName(bucketName)
          .withKey(key)
          .withUploadId(initResponse.getUploadId())
          .withPartNumber(i)
          .withFileOffset(filePosition)
          .withFile(file)
          .withPartSize(partSize);
      partETags.add(s3Client.uploadPart(uploadRequest).getPartETag());
      filePosition += partSize;
    }

    CompleteMultipartUploadRequest completeRequest = new CompleteMultipartUploadRequest(bucketName, key, initResponse.getUploadId(), partETags);
    s3Client.completeMultipartUpload(completeRequest);
  } catch (Exception e) {
    s3Client.abortMultipartUpload(new AbortMultipartUploadRequest(bucketName, key, initResponse.getUploadId()));
    Throwables.propagate(e);
  }
}
 

import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; //导入方法依赖的package包/类
public MultipartUploadOutputStream(AmazonS3 s3, ThreadPoolExecutor threadPool, Progressable progressable, String bucketName, String key, ObjectMetadata metadata, long partSize, File[] tempDirs)
/*     */   {
/*  75 */     RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(4, 10L, TimeUnit.SECONDS);
/*  76 */     Map exceptionToPolicyMap = new HashMap();
/*     */ 
/*  78 */     exceptionToPolicyMap.put(Exception.class, basePolicy);
/*     */ 
/*  80 */     RetryPolicy methodPolicy = RetryPolicies.retryByException(RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
/*     */ 
/*  82 */     Map methodNameToPolicyMap = new HashMap();
/*     */ 
/*  84 */     methodNameToPolicyMap.put("completeMultipartUpload", methodPolicy);
/*     */ 
/*  86 */     this.s3 = ((AmazonS3)RetryProxy.create(AmazonS3.class, s3, methodNameToPolicyMap));
/*  87 */     InitiateMultipartUploadResult result = this.s3.initiateMultipartUpload(new InitiateMultipartUploadRequest(bucketName, key).withObjectMetadata(metadata));
/*     */ 
/*  89 */     this.threadPool = threadPool;
/*  90 */     this.progressable = progressable;
/*  91 */     this.futures = new ArrayList();
/*     */ 
/*  93 */     this.tempDirs = tempDirs;
/*  94 */     this.bucketName = bucketName;
/*  95 */     this.key = key;
/*  96 */     this.uploadId = result.getUploadId();
/*  97 */     this.partSize = partSize;
/*     */ 
/*  99 */     setTempFileAndOutput();
/*     */   }
 

import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; //导入方法依赖的package包/类
private CompleteMultipartUploadResult copyMultipartFile(Bucket srcBucket, Bucket targetBucket, String fileName,
		long size) {
	// Create lists to hold copy responses
	List<CopyPartResult> copyResponses =
	        new ArrayList<CopyPartResult>();

	// Step 2: Initialize
	InitiateMultipartUploadRequest initiateRequest = 
	      	new InitiateMultipartUploadRequest(targetBucket.getName(), targetBucket.getPath() + SEPARATOR + fileName);
	        
	InitiateMultipartUploadResult initResult = 
	       	s3Client.initiateMultipartUpload(initiateRequest);
	
	 // Step 4. Copy parts.
	long partSize = 5 * (long)Math.pow(2.0, 20.0); // 5 MB
	long bytePosition = 0;
	for (int i = 1; bytePosition < size; i++)
	{
	    // Step 5. Save copy response.
		CopyPartRequest copyRequest = new CopyPartRequest()
	       .withDestinationBucketName(targetBucket.getName())
	       .withDestinationKey(targetBucket.getPath() + SEPARATOR + fileName)
	       .withSourceBucketName(srcBucket.getName())
	       .withSourceKey(srcBucket.getPath() + SEPARATOR + fileName)
	       .withUploadId(initResult.getUploadId())
	       .withFirstByte(bytePosition)
	       .withLastByte(bytePosition + partSize -1 >= size ? size - 1 : bytePosition + partSize - 1) 
	       .withPartNumber(i);

	    copyResponses.add(s3Client.copyPart(copyRequest));
	    bytePosition += partSize;
	}
	CompleteMultipartUploadRequest completeRequest = new 
	    	CompleteMultipartUploadRequest(
	    			targetBucket.getName(),
	    			targetBucket.getPath() + SEPARATOR + fileName,
	    			initResult.getUploadId(),
	    			GetETags(copyResponses));
	// Step 7. Complete copy operation.
	CompleteMultipartUploadResult completeUploadResponse =
	    s3Client.completeMultipartUpload(completeRequest);
	return completeUploadResponse;
}
 

import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; //导入方法依赖的package包/类
public static void uploadFile(
        final File file,
        final Artifact artifact,
        final CompressionType compressionType,
        final EncryptionKey encryptionKey,
        final AmazonS3 amazonS3,
        final BuildListener listener) throws IOException {

    LoggingHelper.log(listener, "Uploading artifact: " + artifact + ", file: " + file);

    final String bucketName = artifact.getLocation().getS3Location().getBucketName();
    final String objectKey  = artifact.getLocation().getS3Location().getObjectKey();
    final List<PartETag> partETags = new ArrayList<>();

    final InitiateMultipartUploadRequest initiateMultipartUploadRequest = new InitiateMultipartUploadRequest(
            bucketName,
            objectKey,
            createObjectMetadata(compressionType))
        .withSSEAwsKeyManagementParams(toSSEAwsKeyManagementParams(encryptionKey));

    final InitiateMultipartUploadResult initiateMultipartUploadResult
            = amazonS3.initiateMultipartUpload(initiateMultipartUploadRequest);

    final long contentLength = file.length();
    long filePosition = 0;
    long partSize = 5 * 1024 * 1024; // Set part size to 5 MB

    for (int i = 1; filePosition < contentLength; i++) {
        partSize = Math.min(partSize, (contentLength - filePosition));

        final UploadPartRequest uploadPartRequest = new UploadPartRequest()
                .withBucketName(bucketName)
                .withKey(objectKey)
                .withUploadId(initiateMultipartUploadResult.getUploadId())
                .withPartNumber(i)
                .withFileOffset(filePosition)
                .withFile(file)
                .withPartSize(partSize);

        partETags.add(amazonS3.uploadPart(uploadPartRequest).getPartETag());

        filePosition += partSize;
    }

    final CompleteMultipartUploadRequest completeMultipartUpload
            = new CompleteMultipartUploadRequest(
                bucketName,
                objectKey,
                initiateMultipartUploadResult.getUploadId(),
                partETags);

    amazonS3.completeMultipartUpload(completeMultipartUpload);

    LoggingHelper.log(listener, "Upload successful");
}
 

import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; //导入方法依赖的package包/类
/**
 * Instead of providing support bundle directly to user, upload it to StreamSets backend services.
 */
public void uploadNewBundle(List<String> generators) throws IOException {
  boolean enabled = configuration.get(Constants.UPLOAD_ENABLED, Constants.DEFAULT_UPLOAD_ENABLED);
  String accessKey = configuration.get(Constants.UPLOAD_ACCESS, Constants.DEFAULT_UPLOAD_ACCESS);
  String secretKey = configuration.get(Constants.UPLOAD_SECRET, Constants.DEFAULT_UPLOAD_SECRET);
  String bucket = configuration.get(Constants.UPLOAD_BUCKET, Constants.DEFAULT_UPLOAD_BUCKET);
  int bufferSize = configuration.get(Constants.UPLOAD_BUFFER_SIZE, Constants.DEFAULT_UPLOAD_BUFFER_SIZE);

  if(!enabled) {
    throw new IOException("Uploading support bundles was disabled by administrator.");
  }

  AWSCredentialsProvider credentialsProvider = new StaticCredentialsProvider(new BasicAWSCredentials(accessKey, secretKey));
  AmazonS3Client s3Client = new AmazonS3Client(credentialsProvider, new ClientConfiguration());
  s3Client.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
  s3Client.setRegion(Region.getRegion(Regions.US_WEST_2));

  // Object Metadata
  ObjectMetadata metadata = new ObjectMetadata();
  for(Map.Entry<Object, Object> entry: getMetadata().entrySet()) {
    metadata.addUserMetadata((String)entry.getKey(), (String)entry.getValue());
  }

  // Generate bundle
  SupportBundle bundle = generateNewBundle(generators);

  // Uploading part by part
  LOG.info("Initiating multi-part support bundle upload");
  List<PartETag> partETags = new ArrayList<>();
  InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucket, bundle.getBundleKey());
  initRequest.setObjectMetadata(metadata);
  InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest);

  try {
    byte[] buffer = new byte[bufferSize];
    int partId = 1;
    int size = -1;
    while ((size = readFully(bundle.getInputStream(), buffer)) != -1) {
      LOG.debug("Uploading part {} of size {}", partId, size);
      UploadPartRequest uploadRequest = new UploadPartRequest()
        .withBucketName(bucket)
        .withKey(bundle.getBundleKey())
        .withUploadId(initResponse.getUploadId())
        .withPartNumber(partId++)
        .withInputStream(new ByteArrayInputStream(buffer))
        .withPartSize(size);

      partETags.add(s3Client.uploadPart(uploadRequest).getPartETag());
    }

    CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(
      bucket,
      bundle.getBundleKey(),
      initResponse.getUploadId(),
      partETags
    );

    s3Client.completeMultipartUpload(compRequest);
    LOG.info("Support bundle upload finished");
  } catch (Exception e) {
    LOG.error("Support bundle upload failed", e);
    s3Client.abortMultipartUpload(new AbortMultipartUploadRequest(
      bucket,
      bundle.getBundleKey(),
      initResponse.getUploadId())
    );

    throw new IOException("Can't upload support bundle", e);
  } finally {
    // Close the client
    s3Client.shutdown();
  }
}
 

import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; //导入方法依赖的package包/类
@Test
public void testMultipartCopy() throws Exception {
    // B2 requires two parts to issue an MPU
    assumeTrue(!blobStoreType.equals("b2"));

    String sourceBlobName = "testMultipartCopy-source";
    String targetBlobName = "testMultipartCopy-target";

    ObjectMetadata metadata = new ObjectMetadata();
    metadata.setContentLength(BYTE_SOURCE.size());
    client.putObject(containerName, sourceBlobName,
            BYTE_SOURCE.openStream(), metadata);

    InitiateMultipartUploadRequest initiateRequest =
            new InitiateMultipartUploadRequest(containerName,
                    targetBlobName);
    InitiateMultipartUploadResult initResult =
            client.initiateMultipartUpload(initiateRequest);
    String uploadId = initResult.getUploadId();

    CopyPartRequest copyRequest = new CopyPartRequest()
            .withDestinationBucketName(containerName)
            .withDestinationKey(targetBlobName)
            .withSourceBucketName(containerName)
            .withSourceKey(sourceBlobName)
            .withUploadId(uploadId)
            .withFirstByte(0L)
            .withLastByte(BYTE_SOURCE.size() - 1)
            .withPartNumber(1);
    CopyPartResult copyPartResult = client.copyPart(copyRequest);

    CompleteMultipartUploadRequest completeRequest =
            new CompleteMultipartUploadRequest(
                    containerName, targetBlobName, uploadId,
                    ImmutableList.of(copyPartResult.getPartETag()));
    CompleteMultipartUploadResult completeUploadResponse =
            client.completeMultipartUpload(completeRequest);

    S3Object object = client.getObject(containerName, targetBlobName);
    assertThat(object.getObjectMetadata().getContentLength()).isEqualTo(
            BYTE_SOURCE.size());
    try (InputStream actual = object.getObjectContent();
            InputStream expected = BYTE_SOURCE.openStream()) {
        assertThat(actual).hasContentEqualTo(expected);
    }
}
 

import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; //导入方法依赖的package包/类
@Test
public void testBigMultipartUpload() throws Exception {
    String key = "multipart-upload";
    long partSize = context.getBlobStore().getMinimumMultipartPartSize();
    long size = partSize + 1;
    ByteSource byteSource = TestUtils.randomByteSource().slice(0, size);

    InitiateMultipartUploadRequest initRequest =
            new InitiateMultipartUploadRequest(containerName, key);
    InitiateMultipartUploadResult initResponse =
            client.initiateMultipartUpload(initRequest);
    String uploadId = initResponse.getUploadId();

    ByteSource byteSource1 = byteSource.slice(0, partSize);
    UploadPartRequest uploadRequest1 = new UploadPartRequest()
            .withBucketName(containerName)
            .withKey(key)
            .withUploadId(uploadId)
            .withPartNumber(1)
            .withInputStream(byteSource1.openStream())
            .withPartSize(byteSource1.size());
    uploadRequest1.getRequestClientOptions().setReadLimit(
            (int) byteSource1.size());
    UploadPartResult uploadPartResult1 = client.uploadPart(uploadRequest1);

    ByteSource byteSource2 = byteSource.slice(partSize, size - partSize);
    UploadPartRequest uploadRequest2 = new UploadPartRequest()
            .withBucketName(containerName)
            .withKey(key)
            .withUploadId(uploadId)
            .withPartNumber(2)
            .withInputStream(byteSource2.openStream())
            .withPartSize(byteSource2.size());
    uploadRequest2.getRequestClientOptions().setReadLimit(
            (int) byteSource2.size());
    UploadPartResult uploadPartResult2 = client.uploadPart(uploadRequest2);

    CompleteMultipartUploadRequest completeRequest =
            new CompleteMultipartUploadRequest(
                    containerName, key, uploadId,
                    ImmutableList.of(
                            uploadPartResult1.getPartETag(),
                            uploadPartResult2.getPartETag()));
    client.completeMultipartUpload(completeRequest);

    S3Object object = client.getObject(containerName, key);
    assertThat(object.getObjectMetadata().getContentLength()).isEqualTo(
            size);
    try (InputStream actual = object.getObjectContent();
            InputStream expected = byteSource.openStream()) {
        assertThat(actual).hasContentEqualTo(expected);
    }
}
 

import com.amazonaws.services.s3.model.InitiateMultipartUploadResult; //导入方法依赖的package包/类
public void multipartUpload(String bucketName, String keyName, String filePath) throws IOException {
  //Create a list of UploadPartResponse objects. You get one of these for each part upload.
  List<PartETag> partETags = new ArrayList<PartETag>();

  // Step 1: Initialize.
  InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, keyName);
  InitiateMultipartUploadResult initResponse = s3Client.initiateMultipartUpload(initRequest);

  File file = new File(filePath);
  long contentLength = file.length();
  long partSize = 5 * 1024 * 1024; // Set part size to 1kb.

  try {
    // Step 2: Upload parts.
    long filePosition = 0;
    for (int i = 1; filePosition < contentLength; i++) {
      // Last part can be less than 5 MB. Adjust part size.
      partSize = Math.min(partSize, (contentLength - filePosition));

      // Create request to upload a part.
      UploadPartRequest uploadRequest = new UploadPartRequest()
          .withBucketName(bucketName).withKey(keyName)
          .withUploadId(initResponse.getUploadId()).withPartNumber(i)
          .withFileOffset(filePosition)
          .withFile(file)
          .withPartSize(partSize);

      // Upload part and add response to our list.
      partETags.add(s3Client.uploadPart(uploadRequest).getPartETag());
      filePosition += partSize;
    }

    // Step 3: Complete.
    CompleteMultipartUploadRequest compRequest = 
      new CompleteMultipartUploadRequest(bucketName, keyName, initResponse.getUploadId(), partETags);
    s3Client.completeMultipartUpload(compRequest);
  } catch (Exception e) {
    s3Client.abortMultipartUpload(new AbortMultipartUploadRequest(bucketName, keyName, initResponse.getUploadId()));
    e.printStackTrace();
  }
}