Java ListObjectsRequest.setMarker方法代码示例

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
@Override
@SuppressWarnings("unused")
public List<AwsFileMiniModel> list(String prefix) {

	AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider());
	List<AwsFileMiniModel> files = new ArrayList();

	ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
	ObjectListing objectListing;

	do {
		objectListing = s3client.listObjects(listObjectsRequest);

		for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
			System.out.println(" - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")"
					+ " (date = " + objectSummary.getLastModified() + ")");
			files.add(new AwsFileMiniModel(objectSummary.getKey(), objectSummary.getLastModified()));
		}
		listObjectsRequest.setMarker(objectListing.getNextMarker());
	} while (objectListing.isTruncated());

	return files;

}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
@Override
public List<NoteInfo> list(AuthenticationInfo subject) throws IOException {
  List<NoteInfo> infos = new LinkedList<>();
  NoteInfo info;
  try {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName(bucketName)
            .withPrefix(user + "/" + "notebook");
    ObjectListing objectListing;
    do {
      objectListing = s3client.listObjects(listObjectsRequest);
      for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
        if (objectSummary.getKey().endsWith("note.json")) {
          info = getNoteInfo(objectSummary.getKey());
          if (info != null) {
            infos.add(info);
          }
        }
      }
      listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());
  } catch (AmazonClientException ace) {
    throw new IOException("Unable to list objects in S3: " + ace, ace);
  }
  return infos;
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
@Override
protected void doExecute(Parameters config, Parameters output) {
  String bucketName = this.readAsString(config, "aws.s3.bucketName", null, false, "aws.s3.bucketName not present in config");
  String prefix = this.readAsString(config, "aws.s3.list.prefix", null, false, "aws.s3.list.prefix not present");
  
  AmazonS3 client = new AmazonS3Client(AwsUtils.getCredentialProviderC(config), AwsUtils.getClientConfig(config));
  client.setRegion(AwsUtils.getRegion(config));
  
  ListObjectsRequest listObjectsRequest =
      new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);

  List<S3ObjectSummary> summarys = new ArrayList<S3ObjectSummary>();
  ObjectListing objectListing;
  do {
    objectListing = client.listObjects(listObjectsRequest);
    summarys.addAll(objectListing.getObjectSummaries());
    String marker = objectListing.getNextMarker();
    listObjectsRequest.setMarker(marker);
    
    Detective.info("Reading from S3...current marker:" + marker + " Continue..." );
  } while (objectListing.isTruncated());
  
  output.put("s3ObjectSummaries", summarys);
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
/**
 * Sets public read permissions on content within an S3 bucket.
 * 
 * <p>Web content served from an S3 bucket must have public read permissions.
 * 
 *    @param bucketName the bucket to apply the permissions to.
 *    @param prefix prefix within the bucket, beneath which to apply the permissions.
 *    @param logger a CloudwatchLogs logger.
 */
public static void setPublicReadPermissionsOnBucket(String bucketName, Optional<String> prefix,
    LambdaLogger logger) {
  // Ensure newly uploaded content has public read permission
  ListObjectsRequest listObjectsRequest;
  if (prefix.isPresent()) {
    logger.log("Setting public read permission on bucket: " + bucketName + " and prefix: "
        + prefix.get());
    listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(
        prefix.get());
  } else {
    logger.log("Setting public read permission on bucket: " + bucketName);
    listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName);
  }

  ObjectListing objectListing;
  AmazonS3 client = TransferManagerBuilder.defaultTransferManager().getAmazonS3Client();
  do {
    objectListing = client.listObjects(listObjectsRequest);
    for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
      logger.log("Setting permissions for S3 object: " + objectSummary.getKey());
      client.setObjectAcl(bucketName, objectSummary.getKey(), CannedAccessControlList.PublicRead);
    }
    listObjectsRequest.setMarker(objectListing.getNextMarker());
  } while (objectListing.isTruncated());
  logger.log("Finished setting public read permissions");
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
/**
 * Adds gzip content-encoding metadata to S3 objects.
 * 
 * <p>Adds gzip content-encoding metadata to S3 objects. All objects
 *    beneath the specified prefix (i.e. folder) will have the
 *    metadata added. When the bucket serves objects it will then
 *    add a suitable Content-Encoding header.
 *
 *    @param bucketName the bucket to apply the metadata to.
 *    @param prefix prefix within the bucket, beneath which to apply the metadata.
 *    @param logger a CloudwatchLogs logger.
 */
public static void addGzipContentEncodingMetadata(String bucketName, Optional<String> prefix,
    LambdaLogger logger) {

  // To add new metadata, we must copy each object to itself.
  ListObjectsRequest listObjectsRequest;
  if (prefix.isPresent()) {
    logger.log("Setting gzip content encoding metadata on bucket: " + bucketName
        + " and prefix: " + prefix.get());
    listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(
        prefix.get());
  } else {
    logger.log("Setting gzip content encoding metadata on bucket: " + bucketName);
    listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName);
  }

  ObjectListing objectListing;
  AmazonS3 client = TransferManagerBuilder.defaultTransferManager().getAmazonS3Client();
  do {
    objectListing = client.listObjects(listObjectsRequest);
    for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
      String key = objectSummary.getKey();
      logger.log("Setting metadata for S3 object: " + key);
      // We must specify ALL metadata - not just the one we're adding.
      ObjectMetadata objectMetadata = client.getObjectMetadata(bucketName, key);
      objectMetadata.setContentEncoding("gzip");
      CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucketName, key, bucketName,
          key).withNewObjectMetadata(objectMetadata).withCannedAccessControlList(
          CannedAccessControlList.PublicRead);
      client.copyObject(copyObjectRequest);
      logger.log("Set metadata for S3 object: " + key);
    }
    listObjectsRequest.setMarker(objectListing.getNextMarker());
  } while (objectListing.isTruncated());
  logger.log("Set gzip content encoding metadata on bucket");
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
/**
 * Adds cache-control header to S3 objects.
 * 
 * <p>Adds cache-control header to S3 objects. All objects
 *    beneath the specified prefix (i.e. folder), and with the
 *    specified extension will have the header added. When the
 *    bucket serves objects it will then add a suitable
 *    Cache-Control header.
 *
 *    @param headerValue value of the cache-control header
 *    @param bucketName the bucket to apply the header to.
 *    @param prefix prefix within the bucket, beneath which to apply the header.
 *    @param extension file extension to apply header to
 *    @param logger a CloudwatchLogs logger.
 */
public static void addCacheControlHeader(String headerValue, String bucketName,
    Optional<String> prefix, String extension, LambdaLogger logger) {

  // To add new metadata, we must copy each object to itself.
  ListObjectsRequest listObjectsRequest;
  if (prefix.isPresent()) {
    logger.log("Setting cache-control metadata: " + headerValue + ", on bucket: " + bucketName
        + " and prefix: " + prefix.get() + " and extension: " + extension);
    listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(
        prefix.get());
  } else {
    logger.log("Setting cache-control metadata: " + headerValue + ", on bucket: " + bucketName
        + " and extension: " + extension);
    listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName);
  }

  ObjectListing objectListing;
  AmazonS3 client = TransferManagerBuilder.defaultTransferManager().getAmazonS3Client();
  do {
    objectListing = client.listObjects(listObjectsRequest);
    for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
      String key = objectSummary.getKey();
      if (!key.endsWith(extension)) {
        continue;
      }
      logger.log("Setting metadata for S3 object: " + key);
      // We must specify ALL metadata - not just the one we're adding.
      ObjectMetadata objectMetadata = client.getObjectMetadata(bucketName, key);
      objectMetadata.setCacheControl(headerValue);
      CopyObjectRequest copyObjectRequest = new CopyObjectRequest(bucketName, key, bucketName,
          key).withNewObjectMetadata(objectMetadata).withCannedAccessControlList(
          CannedAccessControlList.PublicRead);
      client.copyObject(copyObjectRequest);
      logger.log("Set metadata for S3 object: " + key);
    }
    listObjectsRequest.setMarker(objectListing.getNextMarker());
  } while (objectListing.isTruncated());
  logger.log("Set cache-control metadata on bucket");
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
@Override
public List<NoteInfo> list() throws IOException {
  List<NoteInfo> infos = new LinkedList<NoteInfo>();
  NoteInfo info = null;
  try {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
        .withBucketName(bucketName)
        .withPrefix(user + "/" + "notebook");
    ObjectListing objectListing;
    do {
      objectListing = s3client.listObjects(listObjectsRequest);

      for (S3ObjectSummary objectSummary :
        objectListing.getObjectSummaries()) {
        if (objectSummary.getKey().contains("note.json")) {
          try {
            info = getNoteInfo(objectSummary.getKey());
            if (info != null) {
              infos.add(info);
            }
          } catch (IOException e) {
            LOG.error("Can't read note ", e);
          }
        }
      }
      listObjectsRequest.setMarker(objectListing.getNextMarker());
    } while (objectListing.isTruncated());
  } catch (AmazonServiceException ase) {

  } catch (AmazonClientException ace) {
    LOG.info("Caught an AmazonClientException, " +
        "which means the client encountered " +
        "an internal error while trying to communicate" +
        " with S3, " +
        "such as not being able to access the network.");
    LOG.info("Error Message: " + ace.getMessage());
  }
  return infos;
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
@Override
public List<FileData> listFileData(String path) {
    List<FileData> ret = new ArrayList<FileData>();
    try {
        String prefix = extraPath + path;
        if (!prefix.endsWith("/")) {
            prefix = prefix + "/";
        }
        prefix = StringUtils.removeStart(prefix, "/");
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
        listObjectsRequest.setDelimiter("/");
        ObjectListing objectListing;
        do {
            objectListing = s3Client.listObjects(listObjectsRequest);
            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
                String fileName = FilenameUtils.getName(FilenameUtils.normalize(objectSummary.getKey()));
                if (StringUtils.isNotBlank(fileName)) {
                    ret.add(new FileData(path, fileName));
                }
            }
            listObjectsRequest.setMarker(objectListing.getNextMarker());
        } while (objectListing.isTruncated());
    } catch (AmazonS3Exception e) {
        LOG.error("Error Listing Files: " + e, e);
        throw new RuntimeException(e);
    }
    return ret;
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
@Override
public void list(String keyPrefix, FileListing callback)
{
    checkArgument(keyPrefix != null, "keyPrefix is null");

    String errorMessage = "listing files on bucket " + bucket + " prefix " + keyPrefix;

    ListObjectsRequest req = new ListObjectsRequest();
    req.setBucketName(bucket);
    req.setPrefix(keyPrefix);

    ObjectListing listing;
    do {
        try {
            listing = getWithRetry(errorMessage, () -> client.listObjects(req));
        }
        catch (StorageFileNotFoundException ex) {
            throw Throwables.propagate(ex.getCause());
        }
        callback.accept(Lists.transform(
                    listing.getObjectSummaries(),
                    (summary) -> StorageObjectSummary.builder()
                        .key(summary.getKey())
                        .contentLength(summary.getSize())
                        .lastModified(summary.getLastModified().toInstant())
                        .build()
                    ));
        req.setMarker(listing.getNextMarker());
    }
    while (listing.isTruncated());
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
public Payload doList(String bucketSuffix, String backendId, WebPath path, Context context) {

		WebPath s3Path = path.addFirst(backendId);
		String bucketName = getBucketName(bucketSuffix);

		ListObjectsRequest request = new ListObjectsRequest()//
				.withBucketName(bucketName)//
				.withPrefix(s3Path.toS3Prefix())//
				.withMaxKeys(context.query().getInteger("size", 100));

		String next = context.get("next");
		if (!Strings.isNullOrEmpty(next))
			request.setMarker(WebPath.parse(next).addFirst(backendId).toS3Key());

		ObjectListing objects = s3.listObjects(request);

		// only root path allows empty list
		if (objects.getObjectSummaries().isEmpty() && s3Path.size() > 1)
			return JsonPayload.error(404);

		JsonBuilder<ObjectNode> response = JsonPayload.builder();

		if (objects.isTruncated())
			response.put("next", fromS3Key(objects.getNextMarker()).toEscapedString());

		response.array("results");

		for (S3ObjectSummary summary : objects.getObjectSummaries()) {
			WebPath objectPath = fromS3Key(summary.getKey());
			response.object()//
					.put("path", objectPath.toString())//
					.put("size", summary.getSize())//
					.put("lastModified", new DateTime(summary.getLastModified().getTime()).toString())//
					.put("etag", summary.getETag())//
					.end();
		}

		return JsonPayload.json(response);
	}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
public List<S3BackupModel> backupList() throws AmazonClientException {
    AwsConn aws = new AwsConn();
    AmazonS3Client s3 = aws.getS3Client(credentials);
    S3Util s3Util = new S3Util(credentials);

    List<S3BackupModel> backupModelList = new ArrayList<S3BackupModel>();
    ObjectListing objectList = null;
    do {
        String prefix = "snapshots/";
        String delimiter = "/";
        if (!prefix.endsWith(delimiter)) {
            prefix += delimiter;
        }

        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix).withDelimiter(delimiter);

        objectList = s3.listObjects(listObjectsRequest);

        for (S3ObjectSummary summary : objectList.getObjectSummaries()) {
            if (! summary.getKey().endsWith("snapshot")) {
                continue;
            }
            
            S3BackupModel backupModel = new S3BackupModel();
            Map<String, String> metadataMap = s3Util.getMetadataForS3Object(bucketName, summary.getKey());

            backupModel.setBackupName(metadataMap.get("backup_name"));
            backupModel.setTimestamp(metadataMap.get("timestamp"));
            backupModel.setDate(metadataMap.get("date"));
            backupModel.setBackupPath(metadataMap.get("backup_path"));

            backupModelList.add(backupModel);

        }
        listObjectsRequest.setMarker(objectList.getNextMarker());
    } while (objectList.isTruncated());

    return backupModelList;
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
public void deleteS3Backup(String backupName, String bucketName) throws AmazonClientException {
    String backupMetadataFileName = "snapshots/" + backupName + ".snapshot";

    ObjectListing objectList = null;
    do {
        ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(backupName);

        objectList = s3.listObjects(listObjectsRequest);
        ;

        if (this.existsS3Object(bucketName, backupMetadataFileName)) {
            s3.deleteObject(bucketName, backupMetadataFileName);
        }

        if (objectList.getObjectSummaries().isEmpty()) {
            LOGGER.warn(String.format("S3 backup " + backupName + " for bucket " + bucketName + " does not exist"));
        }

        for (S3ObjectSummary summary : objectList.getObjectSummaries()) {
            try {
                LOGGER.debug(String.format("Deleting object " + summary.getKey()));
                s3.deleteObject(bucketName, summary.getKey());
            } catch (AmazonClientException e) {
                LOGGER.error(String.format("Delete of object " + summary.getKey() + " failed"));
                LOGGER.error(String.format("S3 backup delete of " + backupName + " failed"));
                throw new AmazonClientException(e.getMessage(), e);
            }
            LOGGER.debug(String.format("S3 object delete for " + summary.getKey() + " completed successfully"));
        }
        listObjectsRequest.setMarker(objectList.getNextMarker());
    } while (objectList.isTruncated());

    LOGGER.info(String.format("S3 backup delete for " + backupName + " completed successfully"));
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
@Override
public List<S3ObjectSummary> findEntityByPrefixKey(String bucketName, String prefixKey) {
	LOG.info("Returns a list of summary information about the objects in the specified bucket "
			+ bucketName + " for the prefix " + prefixKey);
	
	List<S3ObjectSummary> objectSummaries = new ArrayList<S3ObjectSummary>();
	ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
		.withBucketName(bucketName)
		.withPrefix(prefixKey);
	ObjectListing objectListing;
	
	try {
		do {
			objectListing = amazonS3Client.listObjects(listObjectsRequest);
			for (final S3ObjectSummary objectSummary: objectListing.getObjectSummaries()) {
				objectSummaries.add(objectSummary);
			}
			listObjectsRequest.setMarker(objectListing.getNextMarker());
		} while (objectListing.isTruncated());
		LOG.info("Found " + objectSummaries.size()
                + " objects in the specified bucket " + bucketName + " for the prefix " + prefixKey);
	} catch (AmazonServiceException ase) {
		LOG.error("Caught an AmazonServiceException, "
				+ "which means your request made it "
				+ "to Amazon S3, but was rejected with an error response "
				+ "for some reason.", ase);
	} catch (AmazonClientException ace) {
		LOG.error("Caught an AmazonClientException, "
				+ "which means the client encountered "
				+ "an internal error while trying to communicate"
				+ " with S3, "
				+ "such as not being able to access the network.", ace);
	}
       
	return objectSummaries;
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
public static List<S3ObjectSummary> listDirectory(final ClientOptions clientOptions, final String bucketName, final String directory) {
    LOGGER.debug(format("Listing S3 directory %1$s in bucket %2$s", directory, bucketName));

    List<S3ObjectSummary> objects = new ArrayList<>();
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();

    listObjectsRequest.withBucketName(bucketName);
    listObjectsRequest.withPrefix(directory);

    ObjectListing ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest);
    if(ol.isTruncated()) {
        do {
            objects.addAll(ol.getObjectSummaries());
            listObjectsRequest.setMarker(ol.getNextMarker());
            ol = getAmazonS3Client(clientOptions).listObjects(listObjectsRequest);
        } while (ol.isTruncated());
    }
    else {
        objects.addAll(ol.getObjectSummaries());
    }

    if (objects.isEmpty()) {
        return emptyList();
    }

    return unmodifiableList(objects);
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
private List<String> listFiles(String prefix)
{
  List<String> files = new LinkedList<String>();

  try
  {
    AmazonS3 s3Client = new AmazonS3Client(new ClasspathPropertiesFileCredentialsProvider());

    ListObjectsRequest request = new ListObjectsRequest();
    request = request.withBucketName("geodashboarddata");
    request = request.withPrefix(prefix);

    ObjectListing listing;

    do
    {
      listing = s3Client.listObjects(request);

      List<S3ObjectSummary> summaries = listing.getObjectSummaries();

      for (S3ObjectSummary summary : summaries)
      {
        String key = summary.getKey();

        if (key.endsWith(".xml.gz"))
        {
          files.add(key);
        }
      }

      request.setMarker(listing.getNextMarker());
    } while (listing != null && listing.isTruncated());
  }
  catch (Exception e)
  {
    logger.error("Unable to retrieve files", e);
  }

  return files;
}