Java ObjectListing.getNextMarker方法代码示例

import com.amazonaws.services.s3.model.ObjectListing; //导入方法依赖的package包/类
@Override
public void getFileList(String path, OutputStream out) throws Exception {
  String marker = null;
  do {
    ListObjectsRequest request = new ListObjectsRequest(bucketName, path, null, "/", 1000);
    ObjectListing listing = client.listObjects(request);
    for (S3ObjectSummary object : listing.getObjectSummaries()) {
      String line = object.getKey() + "\n";
      out.write(line.getBytes());
    }
    for (String commonPrefix : listing.getCommonPrefixes()) {
      getFileList(commonPrefix, out);
    }
    marker = listing.getNextMarker();
  } while (marker != null);
}
 

import com.amazonaws.services.s3.model.ObjectListing; //导入方法依赖的package包/类
/**
 * Method returns list of all <b>S3ObjectSummary</b> objects, subject to
 * <i>req</i> parameters. Multiple S3 calls will be performed if there are
 * more than 1000 elements there
 * 
 * @param req
 *            - ListObjectRequest to be used.
 * @return List of S3ObjectSummary from bucket,
 */
public List<S3ObjectSummary> getAllSummaries(ListObjectsRequest req) {
    List<S3ObjectSummary> result = new ArrayList<>();
    String marker = null;
    ListObjectsRequest req2 = (ListObjectsRequest) req.clone();
    ObjectListing listing;
    do {
        listing = client.listObjects(req2.withMarker(marker));
        marker = listing.getNextMarker();
        result.addAll(listing.getObjectSummaries());
    } while (listing.isTruncated());

    return result;
}
 

import com.amazonaws.services.s3.model.ObjectListing; //导入方法依赖的package包/类
public static void s3DeleteRecursively(AmazonS3 s3, String bucket, String prefix)
        throws Exception
{
    ListObjectsRequest request = new ListObjectsRequest()
            .withBucketName(bucket)
            .withPrefix(prefix);

    while (true) {
        ObjectListing listing = s3.listObjects(request);
        String[] keys = listing.getObjectSummaries().stream().map(S3ObjectSummary::getKey).toArray(String[]::new);
        for (String key : keys) {
            logger.info("delete s3://{}/{}", bucket, key);
        }
        retryExecutor()
                .retryIf(e -> e instanceof AmazonServiceException)
                .run(() -> s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys)));
        if (listing.getNextMarker() == null) {
            break;
        }
    }
}
 

import com.amazonaws.services.s3.model.ObjectListing; //导入方法依赖的package包/类
@Override
protected void doExecute(Parameters config, Parameters output) {
  String bucketName = this.readAsString(config, "aws.s3.bucketName", null, false, "aws.s3.bucketName not present in config");
  String prefix = this.readAsString(config, "aws.s3.list.prefix", null, false, "aws.s3.list.prefix not present");
  
  AmazonS3 client = new AmazonS3Client(AwsUtils.getCredentialProviderC(config), AwsUtils.getClientConfig(config));
  client.setRegion(AwsUtils.getRegion(config));
  
  ListObjectsRequest listObjectsRequest =
      new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);

  List<S3ObjectSummary> summarys = new ArrayList<S3ObjectSummary>();
  ObjectListing objectListing;
  do {
    objectListing = client.listObjects(listObjectsRequest);
    summarys.addAll(objectListing.getObjectSummaries());
    String marker = objectListing.getNextMarker();
    listObjectsRequest.setMarker(marker);
    
    Detective.info("Reading from S3...current marker:" + marker + " Continue..." );
  } while (objectListing.isTruncated());
  
  output.put("s3ObjectSummaries", summarys);
}
 

import com.amazonaws.services.s3.model.ObjectListing; //导入方法依赖的package包/类
private List<S3ObjectSummary> list(Optional<String> marker) {
    ListObjectsRequest listObjectsRequest = new ListObjectsRequest()
            .withBucketName(bucket)
            .withPrefix(prefix);

    marker.ifPresent(listObjectsRequest::withMarker);

    ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);

    if (objectListing.getObjectSummaries().size() == 0) {
        return Lists.newArrayList();
    }

    List<S3ObjectSummary> collect = objectListing.getObjectSummaries().stream()
            .filter(summary -> summary.getLastModified().getTime() > (System.currentTimeMillis() - notOlderThanTimeUnit.toMillis(notOlderThanValue)))
            .filter(summary -> summary.getKey().endsWith(suffix))
            .collect(Collectors.toList());

    if (collect.size() > 0) {
        return collect;
    } else if (objectListing.getNextMarker() != null) {
        LOGGER.debug("Getting next page with marker {}", objectListing.getNextMarker());
        return list(Optional.of(objectListing.getNextMarker()));
    } else {
        return Lists.newArrayList();
    }
}
 

import com.amazonaws.services.s3.model.ObjectListing; //导入方法依赖的package包/类
/**
 * Gets all tables available in this stash.
 */
public Iterator<StashTable> listTables() {
    final String root = getRootPath();
    final String prefix = String.format("%s/", root);

    return new AbstractIterator<StashTable>() {
        Iterator<String> _commonPrefixes = Iterators.emptyIterator();
        String _marker = null;
        boolean _truncated = true;

        @Override
        protected StashTable computeNext() {
            String dir = null;

            while (dir == null) {
                if (_commonPrefixes.hasNext()) {
                    dir = _commonPrefixes.next();
                    if (dir.isEmpty()) {
                        // Ignore the empty directory if it comes back
                        dir = null;
                    } else {
                        // Strip the prefix and trailing "/"
                        dir = dir.substring(prefix.length(), dir.length()-1);
                    }
                } else if (_truncated) {
                    ObjectListing response = _s3.listObjects(new ListObjectsRequest()
                            .withBucketName(_bucket)
                            .withPrefix(prefix)
                            .withDelimiter("/")
                            .withMarker(_marker)
                            .withMaxKeys(1000));

                    _commonPrefixes = response.getCommonPrefixes().iterator();
                    _marker = response.getNextMarker();
                    _truncated = response.isTruncated();
                } else {
                    return endOfData();
                }
            }

            String tablePrefix = prefix + dir + "/";
            String tableName = StashUtil.decodeStashTable(dir);
            return new StashTable(_bucket, tablePrefix, tableName);
        }
    };
}
 

import com.amazonaws.services.s3.model.ObjectListing; //导入方法依赖的package包/类
/**
 * Gets the metadata for all tables in this stash.  This is a heavier operation that just {@link #listTables()}
 * since it also returns full file details for the entire Stash instead of just table names.
 */
public Iterator<StashTableMetadata> listTableMetadata() {
    final String root = getRootPath();
    final String prefix = String.format("%s/", root);
    final int prefixLength = prefix.length();

    return new AbstractIterator<StashTableMetadata>() {
        PeekingIterator<S3ObjectSummary> _listResponse =
                Iterators.peekingIterator(Iterators.<S3ObjectSummary>emptyIterator());
        String _marker = null;
        boolean _truncated = true;

        @Override
        protected StashTableMetadata computeNext() {
            String tableDir = null;
            List<StashFileMetadata> files = Lists.newArrayListWithCapacity(16);
            boolean allFilesRead = false;

            while (!allFilesRead) {
                if (_listResponse.hasNext()) {
                    // Peek at the next record but don't consume it until we verify it's part of the same table
                    S3ObjectSummary s3File = _listResponse.peek();
                    String key = s3File.getKey();

                    // Don't include the _SUCCESS file or any other stray files we may find
                    String[] parentDirAndFile = key.substring(prefixLength).split("/");
                    if (parentDirAndFile.length != 2) {
                        // Consume and skip this row
                        _listResponse.next();
                    } else {
                        String parentDir = parentDirAndFile[0];
                        if (tableDir == null) {
                            tableDir = parentDir;
                        }

                        if (!parentDir.equals(tableDir)) {
                            allFilesRead = true;
                        } else {
                            // Record is part of this table; consume it now
                            _listResponse.next();
                            files.add(new StashFileMetadata(_bucket, key, s3File.getSize()));
                        }
                    }
                } else if (_truncated) {
                    ObjectListing response = _s3.listObjects(new ListObjectsRequest()
                            .withBucketName(_bucket)
                            .withPrefix(prefix)
                            .withMarker(_marker)
                            .withMaxKeys(1000));

                    _listResponse = Iterators.peekingIterator(response.getObjectSummaries().iterator());
                    _marker = response.getNextMarker();
                    _truncated = response.isTruncated();
                } else {
                    allFilesRead = true;
                }
            }

            if (tableDir == null) {
                // No files read this iteration means all files have been read
                return endOfData();
            }

            String tablePrefix = prefix + tableDir + "/";
            String tableName = StashUtil.decodeStashTable(tableDir);
            return new StashTableMetadata(_bucket, tablePrefix, tableName, files);
        }
    };
}
 

import com.amazonaws.services.s3.model.ObjectListing; //导入方法依赖的package包/类
@Override
protected int poll() throws Exception {
    // must reset for each poll
    shutdownRunningTask = null;
    pendingExchanges = 0;
    
    String fileName = getConfiguration().getFileName();
    String bucketName = getConfiguration().getBucketName();
    Queue<Exchange> exchanges;
    
    if (fileName != null) {
        LOG.trace("Getting object in bucket [{}] with file name [{}]...", bucketName, fileName);

        S3Object s3Object = getAmazonS3Client().getObject(new GetObjectRequest(bucketName, fileName));
        exchanges = createExchanges(s3Object);
    } else {
        LOG.trace("Queueing objects in bucket [{}]...", bucketName);

        ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
        listObjectsRequest.setBucketName(bucketName);
        listObjectsRequest.setPrefix(getConfiguration().getPrefix());
        if (maxMessagesPerPoll > 0) {
            listObjectsRequest.setMaxKeys(maxMessagesPerPoll);
        }
        // if there was a marker from previous poll then use that to continue from where we left last time
        if (marker != null) {
            LOG.trace("Resuming from marker: {}", marker);
            listObjectsRequest.setMarker(marker);
        }

        ObjectListing listObjects = getAmazonS3Client().listObjects(listObjectsRequest);
        if (listObjects.isTruncated()) {
            marker = listObjects.getNextMarker();
            LOG.trace("Returned list is truncated, so setting next marker: {}", marker);
        } else {
            // no more data so clear marker
            marker = null;
        }
        if (LOG.isTraceEnabled()) {
            LOG.trace("Found {} objects in bucket [{}]...", listObjects.getObjectSummaries().size(), bucketName);
        }

        exchanges = createExchanges(listObjects.getObjectSummaries());
    }
    return processBatch(CastUtils.cast(exchanges));
}
 

import com.amazonaws.services.s3.model.ObjectListing; //导入方法依赖的package包/类
@Override
public Integer call() throws Exception {

    BigInteger count = new BigInteger("0");
    String nextMarker = null;
    int nextLimit;
    do {
        // no limit always requests the max
        nextLimit = AWS_MAX_KEYS;
        if(limit != null) {
            // if it is set, limit the next request to no more than the max that are still available
            nextLimit = limit.subtract(count).min(AWS_MAX_KEYS_BIG).intValue();
        }

        if(nextLimit > 0) {
            ObjectListing o = list(nextMarker, nextLimit);
            for(String dir : o.getCommonPrefixes()) {
                printStream.println(String.format(listDirFormat, "DIR", o.getBucketName(), dir));
            }
            for (S3ObjectSummary os : o.getObjectSummaries()) {
                if (limit == null || count.compareTo(limit) <= 0) {
                    printStream.println(String.format(listFormat,
                            os.getLastModified(),
                            os.getSize(),
                            os.getBucketName(),
                            os.getKey(),
                            os.getETag(),
                            os.getOwner(),
                            os.getStorageClass()));
                    nextMarker = o.getNextMarker();
                    count = count.add(ONE);
                } else {
                    // bail out when we've printed all we can
                    nextMarker = null;
                }
            }
        }
    } while (nextMarker != null && nextLimit > 0);

    return 0;
}
 

import com.amazonaws.services.s3.model.ObjectListing; //导入方法依赖的package包/类
private void scanBucket(Set<TocInfo> toc, Queue<TocInfo> tocQueue) throws Exception {
	
	ListObjectsRequest listRequest = new ListObjectsRequest();
	listRequest.setBucketName(s3BucketName);
	// listRequest.setGeneralProgressListener(this);
	listRequest.setMaxKeys(1000);
	
	String nextMarker = null;
	ObjectListing objectListing = null;
	
	while(true) {
		
		objectListing = s3Client.listObjects(listRequest);
		
		List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
		
		for (S3ObjectSummary objSummary : objectSummaries) {
			String key = objSummary.getKey();
			
			TocInfo tocInfo = new TocInfo(key, objSummary.getSize());
			
			// is it a "dir/" ?
			if (key.lastIndexOf("/") == (key.length() - 1)) {
				tocInfo.isDirectory = true;
			} else {
				tocInfo.isDirectory = false;
			}
			
			toc.add(tocInfo);
			tocQueue.add(tocInfo);
			tocInfosGenerated++; // increment for logging

		}
		
		// for pagination
		nextMarker = objectListing.getNextMarker();
		if (nextMarker == null) {
			break;
		} else {
			listRequest.setMarker(nextMarker);
			logger.debug("scanBucket() nextMarker we will request listing for => " + nextMarker);
		}
	}
	
}
 

import com.amazonaws.services.s3.model.ObjectListing; //导入方法依赖的package包/类
public void populateMetadata(DirectoryMetadata directoryMetadata, String bucketName, String prefix) {

        String marker = null;

        do {
            ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucketName, prefix, marker, "/", null);

            ObjectListing objectListing = this.amazonS3Client.listObjects(listObjectsRequest);

            for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {

                if (objectSummary.getKey().equals(prefix)) {
                    continue;
                }

                FileMetadata file = new FileMetadata();

                file.setName(objectSummary.getKey().substring(prefix.length()));
                file.setSizeInBytes(objectSummary.getSize());

                if (objectSummary.getETag().contains("-")) {
                    file.addChecksum(ChecksumType.S3_MULTIPART_ETAG, objectSummary.getETag());
                } else {
                    file.addChecksum(ChecksumType.MD5, objectSummary.getETag());
                }

                directoryMetadata.addFile(file);
            }

            for (String subdirectoryPath : objectListing.getCommonPrefixes()) {

                DirectoryMetadata subdirectory = new DirectoryMetadata();
                subdirectory.setName(subdirectoryPath.substring(prefix.length(), subdirectoryPath.length() - 1));

                directoryMetadata.addSubdirectory(subdirectory);

                populateMetadata(subdirectory, bucketName, subdirectoryPath);
            }

            marker = objectListing.getNextMarker();
        }
        while (marker != null);
    }