Java ListObjectsRequest.setMaxKeys方法代码示例

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
private void prepareCurrentListing() {
    while ( currentListing == null || (!currentIterator.hasNext() && currentListing.isTruncated()) ) {

        if ( currentListing == null ) {
            ListObjectsRequest req = new ListObjectsRequest();
            req.setBucketName(getBucketName());
            req.setPrefix(getPrefix());
            req.setMaxKeys(getBatchSize());
            currentListing = getS3().listObjects(req);
        } else {
            currentListing = getS3().listNextBatchOfObjects(currentListing);
        }

        currentIterator = currentListing.getObjectSummaries().iterator();
    }
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
@Test
public void shouldListObjectWithMaxKeysLimit() throws IOException {
    // Given
    ListObjectsRequest request = new ListObjectsRequest();
    request.setBucketName(bucketName);
    request.setPrefix("COUNTRY_BY_DATE/2014/");
    request.setMaxKeys(2);

    // When
    Observable<ObjectListing> listing = client.listObjects(request);
    ObjectListing amazonListing = amazonS3Client.listObjects(request);

    // Then
    assertThat(listing)
            .ignoreFields(fieldsToIgnore)
            .isEqualTo(amazonListing)
            .isTruncated()
            .hasSize(2);
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
@Test
public void shouldListObjectBatchesWhenUsingRequest() throws IOException {
    // Given
    ListObjectsRequest request = new ListObjectsRequest();
    request.setBucketName(bucketName);
    request.setMaxKeys(2);
    request.setPrefix("COUNTRY_BY_DATE/2014/");

    // When & Then
    Observable<ObjectListing> listing = client.listObjects(request);
    ObjectListing amazonListing = amazonS3Client.listObjects(request);

    while (amazonListing.isTruncated()) {
        assertThat(listing)
                .ignoreFields(fieldsToIgnore)
                .isEqualTo(amazonListing);

        listing = client.listNextBatchOfObjects(listing.toBlocking().single());
        amazonListing = amazonS3Client.listNextBatchOfObjects(amazonListing);
    }

    assertThat(listing)
            .ignoreFields(fieldsToIgnore)
            .isEqualTo(amazonListing)
            .isNotTruncated();
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
@Test
public void shouldListCommonPrefixesInBatches_ContainingFiles() throws IOException {
    // Given
    ListObjectsRequest request = new ListObjectsRequest();
    request.setBucketName(bucketName);
    request.setPrefix("COUNTRY_BY_DATE/2014/05/");
    request.setMaxKeys(1);
    request.setDelimiter("/");

    // When
    ObjectListing amazonListing = amazonS3Client.listObjects(request);
    Observable<ObjectListing> listing = client.listObjects(request);

    // Then
    while (amazonListing.isTruncated()) {
        ObjectListing objectListing = listing.toBlocking().single();

        assertThat(objectListing)
                .ignoreFields(fieldsToIgnore)
                .isEqualTo(amazonListing)
                .isTruncated();

        amazonListing = amazonS3Client.listNextBatchOfObjects(amazonListing);
        listing = client.listNextBatchOfObjects(objectListing);
    }

    assertThat(listing)
            .ignoreFields(fieldsToIgnore)
            .isEqualTo(amazonListing)
            .isNotTruncated();
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
@Test
public void shouldListCommonPrefixesInBatches_ContainingDirectories() throws IOException {
    // Given
    ListObjectsRequest request = new ListObjectsRequest();
    request.setBucketName(bucketName);
    request.setPrefix("COUNTRY_BY_DATE/2014/");
    request.setMaxKeys(1);
    request.setDelimiter("/");

    // When
    Observable<ObjectListing> listing = client.listObjects(request);
    ObjectListing amazonListing = amazonS3Client.listObjects(request);

    // Then
    while (amazonListing.isTruncated()) {
        ObjectListing objectListing = listing.toBlocking().single();

        assertThat(objectListing)
                .ignoreFields(fieldsToIgnore)
                .isEqualTo(amazonListing)
                .isTruncated();

        amazonListing = amazonS3Client.listNextBatchOfObjects(amazonListing);
        listing = client.listNextBatchOfObjects(objectListing);
    }

    assertThat(listing)
            .ignoreFields(fieldsToIgnore)
            .isEqualTo(amazonListing)
            .isNotTruncated();
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
@Override
protected int poll() throws Exception {
    // must reset for each poll
    shutdownRunningTask = null;
    pendingExchanges = 0;
    
    String fileName = getConfiguration().getFileName();
    String bucketName = getConfiguration().getBucketName();
    Queue<Exchange> exchanges;
    
    if (fileName != null) {
        LOG.trace("Getting object in bucket [{}] with file name [{}]...", bucketName, fileName);

        S3Object s3Object = getAmazonS3Client().getObject(new GetObjectRequest(bucketName, fileName));
        exchanges = createExchanges(s3Object);
    } else {
        LOG.trace("Queueing objects in bucket [{}]...", bucketName);

        ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
        listObjectsRequest.setBucketName(bucketName);
        listObjectsRequest.setPrefix(getConfiguration().getPrefix());
        if (maxMessagesPerPoll > 0) {
            listObjectsRequest.setMaxKeys(maxMessagesPerPoll);
        }
        // if there was a marker from previous poll then use that to continue from where we left last time
        if (marker != null) {
            LOG.trace("Resuming from marker: {}", marker);
            listObjectsRequest.setMarker(marker);
        }

        ObjectListing listObjects = getAmazonS3Client().listObjects(listObjectsRequest);
        if (listObjects.isTruncated()) {
            marker = listObjects.getNextMarker();
            LOG.trace("Returned list is truncated, so setting next marker: {}", marker);
        } else {
            // no more data so clear marker
            marker = null;
        }
        if (LOG.isTraceEnabled()) {
            LOG.trace("Found {} objects in bucket [{}]...", listObjects.getObjectSummaries().size(), bucketName);
        }

        exchanges = createExchanges(listObjects.getObjectSummaries());
    }
    return processBatch(CastUtils.cast(exchanges));
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
/**
 * Lists objects from AmazonS3 in lexicographical order
 *
 * @param s3Client
 * @param s3ConfigBean
 * @param pathMatcher glob patterns to match file name against
 * @param s3Offset current offset which provides the key name of the previous object
 * @param fetchSize number of objects to fetch in one go
 * @return
 * @throws AmazonClientException
 */
static List<S3ObjectSummary> listObjectsLexicographically(
    AmazonS3 s3Client,
    S3ConfigBean s3ConfigBean,
    AntPathMatcher pathMatcher,
    AmazonS3Source.S3Offset s3Offset,
    int fetchSize
) {
  // Incrementally scan objects after the marker (s3Offset).
  List<S3ObjectSummary> list = new ArrayList<>(fetchSize);

  ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
  listObjectsRequest.setBucketName(s3ConfigBean.s3Config.bucket);
  listObjectsRequest.setPrefix(s3ConfigBean.s3Config.commonPrefix);
  listObjectsRequest.setMaxKeys(BATCH_SIZE);
  if (s3Offset.getKey() != null) {
    listObjectsRequest.setMarker(s3Offset.getKey());
  }

  ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);

  while (true) {
    for (S3ObjectSummary s : objectListing.getObjectSummaries()) {
      String fullPrefix = s.getKey();
      String remainingPrefix = fullPrefix.substring(s3ConfigBean.s3Config.commonPrefix.length(), fullPrefix.length());
      if (!remainingPrefix.isEmpty()) {
        if (pathMatcher.match(s3ConfigBean.s3FileConfig.prefixPattern, remainingPrefix)) {
          list.add(s);
        }
        // We've got enough objects.
        if (list.size() == fetchSize) {
          return list;
        }
      }
    }
    // Listing is complete. No more objects to be listed.
    if (!objectListing.isTruncated()) {
      break;
    }
    objectListing = s3Client.listNextBatchOfObjects(objectListing);
  }

  return list;
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
/**
 * Calculate the total size of all objects in the indicated bucket
 *
 * @param path to use
 * @return calculated size
 * @throws IOException
 */
@Override
public long calculateSize(Path path) throws IOException {
  long totalBucketSize = 0L;

  if (s3Enabled) {
    String key = pathToKey(path);

    final FileStatus fileStatus =  fs.getFileStatus(path);

    if (fileStatus.isDirectory()) {
      if (!key.isEmpty()) {
        key = key + "/";
      }

      ListObjectsRequest request = new ListObjectsRequest();
      request.setBucketName(uri.getHost());
      request.setPrefix(key);
      request.setMaxKeys(maxKeys);

      if (LOG.isDebugEnabled()) {
        LOG.debug("listStatus: doing listObjects for directory " + key);
      }

      ObjectListing objects = s3.listObjects(request);

      while (true) {
        for (S3ObjectSummary summary : objects.getObjectSummaries()) {
          Path keyPath = keyToPath(summary.getKey()).makeQualified(uri, fs.getWorkingDirectory());

          // Skip over keys that are ourselves and old S3N _$folder$ files
          if (keyPath.equals(path) || summary.getKey().endsWith(S3N_FOLDER_SUFFIX)) {
            if (LOG.isDebugEnabled()) {
              LOG.debug("Ignoring: " + keyPath);
            }
            continue;
          }

          if (!objectRepresentsDirectory(summary.getKey(), summary.getSize())) {
            totalBucketSize += summary.getSize();
          }
        }

        if (objects.isTruncated()) {
          if (LOG.isDebugEnabled()) {
            LOG.debug("listStatus: list truncated - getting next batch");
          }
          objects = s3.listNextBatchOfObjects(objects);
        } else {
          break;
        }
      }
    } else {
      return fileStatus.getLen();
    }
  } else {
    totalBucketSize = fs.getContentSummary(path).getLength();
  }

  return totalBucketSize;
}
 

import com.amazonaws.services.s3.model.ListObjectsRequest; //导入方法依赖的package包/类
private void scanBucket(Set<TocInfo> toc, Queue<TocInfo> tocQueue) throws Exception {
	
	ListObjectsRequest listRequest = new ListObjectsRequest();
	listRequest.setBucketName(s3BucketName);
	// listRequest.setGeneralProgressListener(this);
	listRequest.setMaxKeys(1000);
	
	String nextMarker = null;
	ObjectListing objectListing = null;
	
	while(true) {
		
		objectListing = s3Client.listObjects(listRequest);
		
		List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
		
		for (S3ObjectSummary objSummary : objectSummaries) {
			String key = objSummary.getKey();
			
			TocInfo tocInfo = new TocInfo(key, objSummary.getSize());
			
			// is it a "dir/" ?
			if (key.lastIndexOf("/") == (key.length() - 1)) {
				tocInfo.isDirectory = true;
			} else {
				tocInfo.isDirectory = false;
			}
			
			toc.add(tocInfo);
			tocQueue.add(tocInfo);
			tocInfosGenerated++; // increment for logging

		}
		
		// for pagination
		nextMarker = objectListing.getNextMarker();
		if (nextMarker == null) {
			break;
		} else {
			listRequest.setMarker(nextMarker);
			logger.debug("scanBucket() nextMarker we will request listing for => " + nextMarker);
		}
	}
	
}