Java S3ObjectSummary.getKey方法代码示例

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入方法依赖的package包/类
public void deleteAll(String bucket) {
    logger.info("delete all from bucket, bucket={}", bucket);

    ObjectListing listing = s3.listObjects(new ListObjectsRequest().withBucketName(bucket));

    while (listing != null) {
        List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>(listing.getObjectSummaries().size());
        for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            String key = summary.getKey();
            logger.info("add key to deletion batch, key={}", key);
            keys.add(new DeleteObjectsRequest.KeyVersion(key));
        }
        if (!keys.isEmpty()) {
            logger.info("delete key batch");
            s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys));
        }
        if (!listing.isTruncated()) return;

        listing = s3.listNextBatchOfObjects(listing);
    }
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入方法依赖的package包/类
@Override
public void getFileList(String path, OutputStream out) throws Exception {
  String marker = null;
  do {
    ListObjectsRequest request = new ListObjectsRequest(bucketName, path, null, "/", 1000);
    ObjectListing listing = client.listObjects(request);
    for (S3ObjectSummary object : listing.getObjectSummaries()) {
      String line = object.getKey() + "\n";
      out.write(line.getBytes());
    }
    for (String commonPrefix : listing.getCommonPrefixes()) {
      getFileList(commonPrefix, out);
    }
    marker = listing.getNextMarker();
  } while (marker != null);
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入方法依赖的package包/类
@Override
public Iterable<BinaryKey> getAllBinaryKeys() throws BinaryStoreException {
    try {
        final Iterator<S3ObjectSummary> objectsIterator =
            S3Objects.inBucket(s3Client, bucketName).iterator();
        // Lambda to hand back BinaryKeys rather than S3ObjectSummaries
        return () -> {
            return new Iterator<BinaryKey>() {
                @Override
                public boolean hasNext() {
                    return objectsIterator.hasNext();
                }

                @Override
                public BinaryKey next() {
                    S3ObjectSummary object = objectsIterator.next();
                    return new BinaryKey(object.getKey());
                }
            };
        };
    } catch (AmazonClientException e) {
        throw new BinaryStoreException(e);
    }
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入方法依赖的package包/类
public static String getS3KeyFromBucket(String filenamePrefix) {
	try {
		ObjectListing ol = S3.listObjects(properties.getProperty(S3_BUCKET_NAME));
		List<S3ObjectSummary> objects = ol.getObjectSummaries();
		for (S3ObjectSummary os : objects) {
			if (StringUtils.startsWith(os.getKey(), filenamePrefix)) {
				LOG.info("Found file {} from S3 bucket.", filenamePrefix);
				return os.getKey();
			}
		}
	} catch (AmazonServiceException e) {
		LOG.error("Exception occured when checking if file exists in S3.", e);
	}
	return StringUtils.EMPTY;
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入方法依赖的package包/类
private void submitCopyJobsFromListing(
    AmazonS3URI sourceS3Uri,
    final AmazonS3URI targetS3Uri,
    ListObjectsRequest request,
    ObjectListing listing) {
  LOG.debug("Found objects to copy {}, for request {}/{}", listing.getObjectSummaries(), request.getBucketName(),
      request.getPrefix());
  List<S3ObjectSummary> objectSummaries = listing.getObjectSummaries();
  for (final S3ObjectSummary s3ObjectSummary : objectSummaries) {
    String fileName = StringUtils.removeStart(s3ObjectSummary.getKey(), sourceS3Uri.getKey());
    final String targetKey = Strings.nullToEmpty(targetS3Uri.getKey()) + fileName;
    LOG.info("copying object from '{}/{}' to '{}/{}'", s3ObjectSummary.getBucketName(), s3ObjectSummary.getKey(),
        targetS3Uri.getBucket(), targetKey);

    CopyObjectRequest copyObjectRequest = new CopyObjectRequest(s3ObjectSummary.getBucketName(),
        s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey);

    TransferStateChangeListener stateChangeListener = new TransferStateChangeListener() {

      @Override
      public void transferStateChanged(Transfer transfer, TransferState state) {
        if (state == TransferState.Completed) {
          // NOTE: running progress doesn't seem to be reported correctly.
          // transfer.getProgress().getBytesTransferred() is always 0. Unsure what is the cause of this at this moment
          // so just printing total bytes when completed.
          LOG.debug("copied object from '{}/{}' to '{}/{}': {} bytes transferred", s3ObjectSummary.getBucketName(),
              s3ObjectSummary.getKey(), targetS3Uri.getBucket(), targetKey,
              transfer.getProgress().getTotalBytesToTransfer());
        }
      }
    };
    Copy copy = transferManager.copy(copyObjectRequest, srcClient, stateChangeListener);
    totalBytesToReplicate += copy.getProgress().getTotalBytesToTransfer();
    copyJobs.add(copy);
  }
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入方法依赖的package包/类
@Override
public Collection<String> doInBackground(String... path) {
    List<String> listItems = new ArrayList<>();
    String prefix = path[0];
    try {
        MainActivity.this.listing = client.listObjects(new ListObjectsRequest()
                .withBucketName(preferences.bucketName)
                .withDelimiter("/")
                .withPrefix(prefix));
    } catch (AmazonClientException ace) {
        Log.e(TAG, "Error listing with prefix: " + prefix + " " + ace.getMessage());
        return null;
    }
    for (S3ObjectSummary summary : MainActivity.this.listing.getObjectSummaries()) {
        String key = summary.getKey();
        Log.d(TAG, "listing key: " + key);
        if (key.equals(prefix)) {
            continue;
        }
        listItems.add(key.substring(prefix.length()));
    }
    for (String commonPrefix : MainActivity.this.listing.getCommonPrefixes()) {
        Log.d(TAG, "listing common prefix: " + commonPrefix);
        listItems.add(commonPrefix.substring(prefix.length()));
    }
    Collections.sort(listItems);
    return listItems;
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入方法依赖的package包/类
private void addKeyUris(List<URI> uris, ObjectListing list, URI uri, Predicate<URI> uriPredicate) {
    List<S3ObjectSummary> summaries = list.getObjectSummaries();
    for (S3ObjectSummary summary : summaries) {
        String key = summary.getKey();
        if (!key.endsWith("/")) {
            URI keyUri = uri.resolve("/" + key);
            if (uriPredicate.apply(keyUri)) {
                uris.add(keyUri);
                if (logger.isDebugEnabled()) {
                    logger.debug("{}", keyUri);
                }
            }
        }
    }
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入方法依赖的package包/类
private FileStatus createFileStatus(S3ObjectSummary objSummary,
    String hostName, Path path)
    throws IllegalArgumentException, IOException {
  String objKey = objSummary.getKey();
  String newMergedPath = getMergedPath(hostName, path, objKey);
  return createFileStatus(objSummary.getSize(), objKey,
      objSummary.getLastModified(), new Path(newMergedPath));
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入方法依赖的package包/类
/**
 * Adds gzip content-encoding metadata to S3 objects.
 * 
 * <p>Adds gzip content-encoding metadata to S3 objects. All objects
 *    beneath the specified prefix (i.e. folder) will have the
 *    metadata added. When the bucket serves objects it will then
 *    add a suitable Content-Encoding header.
 *
 *    @param bucketName the bucket to apply the metadata to.
 *    @param prefix prefix within the bucket, beneath which to apply the metadata.
 *    @param logger a CloudwatchLogs logger.
 */
public static void addGzipContentEncodingMetadata(String bucketName, Optional<String> prefix,
    LambdaLogger logger) {

  // To add new metadata, we must copy each object to itself.
  ListObjectsRequest listObjectsRequest;
  if (prefix.isPresent()) {
    logger.log("Setting gzip content encoding metadata on bucket: " + bucketName
        + " and prefix: " + prefix.get());
    listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(
        prefix.get());
  } else {
    logger.log("Setting gzip content encoding metadata on bucket: " + bucketName);
    listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName);
  }

  ObjectListing objectListing;
  AmazonS3 client = TransferManagerBuilder.defaultTransferManager().getAmazonS3Client();
  do {
    objectListing = client.listObjects(listObjectsRequest);
    for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
      String key = objectSummary.getKey();
      logger.log("Setting metadata for S3 object: " + key);
      // We must specify ALL metadata - not just the one we're adding.
      ObjectMetadata objectMetadata = client.getObjectMetadata(bucketName, key);
      objectMetadata.setContentEncoding("gzip");
      CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucketName, key, bucketName,
          key).withNewObjectMetadata(objectMetadata).withCannedAccessControlList(
          CannedAccessControlList.PublicRead);
      client.copyObject(copyObjectRequest);
      logger.log("Set metadata for S3 object: " + key);
    }
    listObjectsRequest.setMarker(objectListing.getNextMarker());
  } while (objectListing.isTruncated());
  logger.log("Set gzip content encoding metadata on bucket");
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入方法依赖的package包/类
/**
 * Adds cache-control header to S3 objects.
 * 
 * <p>Adds cache-control header to S3 objects. All objects
 *    beneath the specified prefix (i.e. folder), and with the
 *    specified extension will have the header added. When the
 *    bucket serves objects it will then add a suitable
 *    Cache-Control header.
 *
 *    @param headerValue value of the cache-control header
 *    @param bucketName the bucket to apply the header to.
 *    @param prefix prefix within the bucket, beneath which to apply the header.
 *    @param extension file extension to apply header to
 *    @param logger a CloudwatchLogs logger.
 */
public static void addCacheControlHeader(String headerValue, String bucketName,
    Optional<String> prefix, String extension, LambdaLogger logger) {

  // To add new metadata, we must copy each object to itself.
  ListObjectsRequest listObjectsRequest;
  if (prefix.isPresent()) {
    logger.log("Setting cache-control metadata: " + headerValue + ", on bucket: " + bucketName
        + " and prefix: " + prefix.get() + " and extension: " + extension);
    listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(
        prefix.get());
  } else {
    logger.log("Setting cache-control metadata: " + headerValue + ", on bucket: " + bucketName
        + " and extension: " + extension);
    listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName);
  }

  ObjectListing objectListing;
  AmazonS3 client = TransferManagerBuilder.defaultTransferManager().getAmazonS3Client();
  do {
    objectListing = client.listObjects(listObjectsRequest);
    for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
      String key = objectSummary.getKey();
      if (!key.endsWith(extension)) {
        continue;
      }
      logger.log("Setting metadata for S3 object: " + key);
      // We must specify ALL metadata - not just the one we're adding.
      ObjectMetadata objectMetadata = client.getObjectMetadata(bucketName, key);
      objectMetadata.setCacheControl(headerValue);
      CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucketName, key, bucketName,
          key).withNewObjectMetadata(objectMetadata).withCannedAccessControlList(
          CannedAccessControlList.PublicRead);
      client.copyObject(copyObjectRequest);
      logger.log("Set metadata for S3 object: " + key);
    }
    listObjectsRequest.setMarker(objectListing.getNextMarker());
  } while (objectListing.isTruncated());
  logger.log("Set cache-control metadata on bucket");
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入方法依赖的package包/类
/**
  * Contructs a StorageObjectSummary object from the S3 equivalent S3ObjectSummary
  * @param objSummary the AWS S3 ObjectSummary object to copy from
  * @return the ObjectSummary object created
  */
public static StorageObjectSummary createFromS3ObjectSummary(S3ObjectSummary objSummary)
 {

   return new StorageObjectSummary(
                 objSummary.getBucketName(),
                 objSummary.getKey(),
                 // S3 ETag is not always MD5, but since this code path is only
                 // used in skip duplicate files in PUT command, It's not
                 // critical to guarantee that it's MD5
                 objSummary.getETag(),
                 objSummary.getSize()
                 );
 }
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入方法依赖的package包/类
/**
 * Get the splits for a record stored in stash.  Each split corresponds to a file in the Stash table's directory.
 */
public List<StashSplit> getSplits(String table)
        throws StashNotAvailableException, TableNotStashedException {
    ImmutableList.Builder<StashSplit> splitsBuilder = ImmutableList.builder();

    Iterator<S3ObjectSummary> objectSummaries = getS3ObjectSummariesForTable(table);
    while (objectSummaries.hasNext()) {
        S3ObjectSummary objectSummary = objectSummaries.next();
        String key = objectSummary.getKey();
        // Strip the common root path prefix from the split since it is constant.
        splitsBuilder.add(new StashSplit(table, key.substring(_rootPath.length() + 1), objectSummary.getSize()));
    }

    return splitsBuilder.build();
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入方法依赖的package包/类
protected void verifyFileListing(List<String> expectedFiles) throws IOException {
  List<S3ObjectSummary> summaries = listObjects(S3_TEST_BUCKET_NAME, null, s3);
  List<String> actualFiles = new ArrayList<>();
  for (S3ObjectSummary summary : summaries) {
    String fileKey = summary.getKey();
    actualFiles.add(fileKey);
  }

  Collections.sort(actualFiles);
  Collections.sort(expectedFiles);
  assertThat(actualFiles, is(expectedFiles));
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入方法依赖的package包/类
@Override
public List<RemoteFile> getChildren(String folderName) throws IOException {
    if (!folderName.endsWith("/")) {
        folderName = folderName + "/";
    }

    ListObjectsRequest listRequest = new ListObjectsRequest();
    listRequest.setBucketName(getBucketName());
    listRequest.setDelimiter("/");
    listRequest.setPrefix(folderName);

    ObjectListing listing = s3.listObjects(listRequest);

    ArrayList<RemoteFile> list = new ArrayList<>();

    for (S3ObjectSummary summ : listing.getObjectSummaries()) {
        String name = summ.getKey();
        long size = summ.getSize();

        boolean isDirectory = isFolder(name);

        RemoteFile file = new RemoteFile(name, isDirectory, size);
        list.add(file);
    }

    return list;
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入方法依赖的package包/类
private int verifyAndReturnNoOfObjects() throws Exception {
  int numberOfObjects = 0;
  for (S3ObjectSummary s3ObjectSummary : S3Objects.inBucket(s3client, TARGET_BUCKET_NAME)) {
    String fileNameOrKey = s3ObjectSummary.getKey();
    if (withFileNamePrefix) {
      //strip out the filePrefix sdc-
      fileNameOrKey = fileNameOrKey.substring(4);
    }
    switch (source) {
      case LOCAL:
        verifyStreamCorrectness(
            new FileInputStream(testDir.getAbsolutePath() + "/" + fileNameOrKey),
            s3client.getObject(TARGET_BUCKET_NAME, s3ObjectSummary.getKey()).getObjectContent()
        );
        break;
      case S3:
        verifyStreamCorrectness(
            s3client.getObject(SOURCE_BUCKET_NAME, fileNameOrKey).getObjectContent(),
            s3client.getObject(TARGET_BUCKET_NAME, s3ObjectSummary.getKey()).getObjectContent()
        );
        break;
    }
    deleteObjectsAfterVerificationInTarget(s3ObjectSummary.getKey());
    numberOfObjects++;
  }
  return numberOfObjects;
}