Java S3ObjectSummary类代码示例

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入依赖的package包/类
public void deleteAll(String bucket) {
    logger.info("delete all from bucket, bucket={}", bucket);

    ObjectListing listing = s3.listObjects(new ListObjectsRequest().withBucketName(bucket));

    while (listing != null) {
        List<DeleteObjectsRequest.KeyVersion> keys = new ArrayList<>(listing.getObjectSummaries().size());
        for (S3ObjectSummary summary : listing.getObjectSummaries()) {
            String key = summary.getKey();
            logger.info("add key to deletion batch, key={}", key);
            keys.add(new DeleteObjectsRequest.KeyVersion(key));
        }
        if (!keys.isEmpty()) {
            logger.info("delete key batch");
            s3.deleteObjects(new DeleteObjectsRequest(bucket).withKeys(keys));
        }
        if (!listing.isTruncated()) return;

        listing = s3.listNextBatchOfObjects(listing);
    }
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入依赖的package包/类
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
    if ("nonExistingBucket".equals(listObjectsRequest.getBucketName()) && !nonExistingBucketCreated) {
        AmazonServiceException ex = new AmazonServiceException("Unknown bucket");
        ex.setStatusCode(404);
        throw ex; 
    }
    int capacity;
    ObjectListing objectListing = new ObjectListing();
    if (!ObjectHelper.isEmpty(listObjectsRequest.getMaxKeys()) && listObjectsRequest.getMaxKeys() != null) {
        capacity = listObjectsRequest.getMaxKeys();
    } else {
        capacity = maxCapacity;
    }
    
    for (int index = 0; index < objects.size() && index < capacity; index++) {
        S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
        s3ObjectSummary.setBucketName(objects.get(index).getBucketName());
        s3ObjectSummary.setKey(objects.get(index).getKey());
        
        objectListing.getObjectSummaries().add(s3ObjectSummary);
    }

    return objectListing;
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入依赖的package包/类
private boolean isUploadFile(Iterator<S3ObjectSummary> iter, String path, String hash) {
    while (iter.hasNext()) {
        S3ObjectSummary fileS3 = iter.next();
        // Filename should look like this:
        // a/b
        if (!fileS3.getKey().equals(path)) {
            // If this is another file, then continue!
            continue;
        }
        // Remove the file from the S3 list as it does not need to be processed further
        iter.remove();
        // Upload if the hashes differ
        return StringUtils.isNullOrEmpty(hash) || !fileS3.getETag().equals(hash);
    }
    return true;
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入依赖的package包/类
@Override
public Stream<WriteableConfiguration> getCachedConfigurations() throws IOException {

  Iterable<S3ObjectSummary> objectSummaries = S3Objects.withPrefix(s3, bucket, prefix);
  Stream<S3ObjectSummary> objectStream = StreamSupport.stream(objectSummaries.spliterator(), false);

  return objectStream.map(p -> {
    Integer version = getVersionIfMatch(p.getKey());
    if (version == null) {
      return null;
    }
    return new Pair<>(version, p);
  }).filter(Objects::nonNull)
      .sorted(Comparator.comparing(pair -> ((Pair<Integer, S3ObjectSummary>) pair).getFirst())
            .reversed()).map(pair -> new S3WritableConfiguration(s3, pair.getSecond(), Integer.toString(pair.getFirst())));

}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入依赖的package包/类
@Override
public void getFileList(String path, OutputStream out) throws Exception {
  String marker = null;
  do {
    ListObjectsRequest request = new ListObjectsRequest(bucketName, path, null, "/", 1000);
    ObjectListing listing = client.listObjects(request);
    for (S3ObjectSummary object : listing.getObjectSummaries()) {
      String line = object.getKey() + "\n";
      out.write(line.getBytes());
    }
    for (String commonPrefix : listing.getCommonPrefixes()) {
      getFileList(commonPrefix, out);
    }
    marker = listing.getNextMarker();
  } while (marker != null);
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入依赖的package包/类
private boolean processObjects(List<S3ObjectSummary> objects) {
    Logger.Debug("Scanning next batch of %s ", objects.size());
    objects
        .parallelStream()
        .filter(this::shouldEnqueue)
        .forEach(object -> {
            numSeen.incrementAndGet();
            String path = object.getBucketName() + "/" + object.getKey();
            Logger.Info("Posting: %s", path);

            SendMessageRequest msg = new SendMessageRequest()
                .withQueueUrl(queueUrl)
                .withMessageBody(path);
            sqs.sendMessage(msg);
        });
    if (max > -1L && numSeen.incrementAndGet() > max) {
        Logger.Info("Added max jobs, quitting");
        return false;
    }
    return true;
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入依赖的package包/类
@Override
@SuppressWarnings("unused")
public List<AwsFileMiniModel> list(String prefix) {

	AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider());
	List<AwsFileMiniModel> files = new ArrayList();

	ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(prefix);
	ObjectListing objectListing;

	do {
		objectListing = s3client.listObjects(listObjectsRequest);

		for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) {
			System.out.println(" - " + objectSummary.getKey() + "  " + "(size = " + objectSummary.getSize() + ")"
					+ " (date = " + objectSummary.getLastModified() + ")");
			files.add(new AwsFileMiniModel(objectSummary.getKey(), objectSummary.getLastModified()));
		}
		listObjectsRequest.setMarker(objectListing.getNextMarker());
	} while (objectListing.isTruncated());

	return files;

}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入依赖的package包/类
/**
 * Test cleanup deletes everything in the bucket so obviously only use on Pairtree test buckets.
 *
 * @param aContext A test context
 */
@Override
@After
public void tearDown(final TestContext aContext) {
    super.tearDown(aContext);

    // Clean up our test resources in the S3 bucket
    final ObjectListing listing = myS3Client.listObjects(myTestBucket);
    final Iterator<S3ObjectSummary> iterator = listing.getObjectSummaries().iterator();

    while (iterator.hasNext()) {
        try {
            myS3Client.deleteObject(myTestBucket, iterator.next().getKey());
        } catch (final AmazonClientException details) {
            aContext.fail(details);
        }
    }
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入依赖的package包/类
static void deletePrefix(AmazonS3 s3Client, String bucketName, String prefix) {
    List<DeleteObjectsRequest.KeyVersion> trashKeys = new ArrayList<>();
    DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(bucketName);
    for (S3ObjectSummary summary: S3Objects.withPrefix(s3Client, bucketName, prefix)) {
        trashKeys.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
        if (trashKeys.size() == BULK_DELETE_SIZE) {
            deleteObjectsRequest.setKeys(trashKeys);
            s3Client.deleteObjects(deleteObjectsRequest);
            trashKeys.clear();
        }
    }
    if (!trashKeys.isEmpty()) {
        deleteObjectsRequest.setKeys(trashKeys);
        s3Client.deleteObjects(deleteObjectsRequest);
    }
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入依赖的package包/类
@Override
protected void copy(RuntimeExecutionTrace source, RuntimeExecutionTrace target,
        RuntimeAnnotatedExecutionTrace absoluteSource, RuntimeAnnotatedExecutionTrace absoluteTarget)
        throws IOException {
    String sourcePrefix = toS3Path(source).getPrefixForChildren();
    String targetPrefix = toS3Path(target).getPrefixForChildren();
    for (S3ObjectSummary summary: S3Objects.withPrefix(s3Client, bucketName, sourcePrefix)) {
        if (summary.getKey().startsWith(sourcePrefix)) {
            String relativeKey = summary.getKey().substring(sourcePrefix.length());
            s3Client.copyObject(bucketName, summary.getKey(), bucketName, targetPrefix + relativeKey);
        } else {
            log.error(String.format(
                "S3Objects.withPrefix() returned unexpected key '%s' when asked for prefix '%s'.",
                summary.getKey(), sourcePrefix
            ));
        }
    }
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入依赖的package包/类
@Override
protected void tearDown() throws Exception {
	final File theFile = new File(TEST_FILE_NAME);
	theFile.delete();

	if (s3Account != null) {
		final AmazonS3Client s3 = new AmazonS3Client(
				new BasicAWSCredentials(s3Account.keyId, s3Account.secretKey));
		String hash = s3Account.getHashedName();
		if (s3.doesBucketExist(hash)) {
			for (S3ObjectSummary summary : s3.listObjects(hash).getObjectSummaries()) {
				s3.deleteObject(s3Account.getHashedName(), summary.getKey());
			}

		}
	}

	super.tearDown();
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入依赖的package包/类
/**
 * Gets the metadata for a single table in this stash.  This is similar to getting the splits for the table
 * except that it exposes lower level information about the underlying S3 files.  For clients who will use
 * their own system for reading the files from S3, such as source files for a map-reduce job, this method provides
 * the necessary information.  For simply iterating over the stash contents using either {@link #scan(String)}
 * or {@link #getSplits(String)} in conjunction with {@link #getSplit(StashSplit)} is preferred.
 */
public StashTableMetadata getTableMetadata(String table)
        throws StashNotAvailableException, TableNotStashedException {
    ImmutableList.Builder<StashFileMetadata> filesBuilder = ImmutableList.builder();

    Iterator<S3ObjectSummary> objectSummaries = getS3ObjectSummariesForTable(table);
    while (objectSummaries.hasNext()) {
        S3ObjectSummary objectSummary = objectSummaries.next();
        filesBuilder.add(new StashFileMetadata(_bucket, objectSummary.getKey(), objectSummary.getSize()));
    }

    List<StashFileMetadata> files = filesBuilder.build();

    // Get the prefix arbitrarily from the first file.
    String prefix = files.get(0).getKey();
    prefix = prefix.substring(0, prefix.lastIndexOf('/') + 1);

    return new StashTableMetadata(_bucket, prefix, table, files);
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入依赖的package包/类
private Answer<ObjectListing> objectListingAnswer(@Nullable final String marker, final String... fileNames) {
    return new Answer<ObjectListing>() {
        @Override
        public ObjectListing answer(InvocationOnMock invocation)
                throws Throwable {
            ListObjectsRequest request = (ListObjectsRequest) invocation.getArguments()[0];

            ObjectListing objectListing = new ObjectListing();
            objectListing.setBucketName(request.getBucketName());
            objectListing.setPrefix(request.getPrefix());

            objectListing.setTruncated(marker != null);
            objectListing.setNextMarker(marker);

            for (String fileName : fileNames) {
                S3ObjectSummary objectSummary = new S3ObjectSummary();
                objectSummary.setKey(request.getPrefix() + fileName);
                objectSummary.setSize(100);
                objectListing.getObjectSummaries().add(objectSummary);
            }

            return objectListing;
        }
    };
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入依赖的package包/类
@Test
public void testCompressFile() throws Exception {
  String avroCodec = "snappy";
  localProps.put(StorageSinkConnectorConfig.AVRO_CODEC_CONFIG, avroCodec);
  setUp();
  task = new S3SinkTask(connectorConfig, context, storage, partitioner, format, SYSTEM_TIME);

  List<SinkRecord> sinkRecords = createRecords(7);
  // Perform write
  task.put(sinkRecords);
  task.close(context.assignment());
  task.stop();

  List<S3ObjectSummary> summaries = listObjects(S3_TEST_BUCKET_NAME, "/", s3);
  for(S3ObjectSummary summary: summaries){
    InputStream in = s3.getObject(summary.getBucketName(), summary.getKey()).getObjectContent();
    DatumReader<Object> reader = new GenericDatumReader<>();
    DataFileStream<Object> streamReader = new DataFileStream<>(in, reader);
    // make sure that produced Avro file has proper codec set
    Assert.assertEquals(avroCodec, streamReader.getMetaString(StorageSinkConnectorConfig.AVRO_CODEC_CONFIG));
    streamReader.close();
  }

  long[] validOffsets = {0, 3, 6};
  verify(sinkRecords, validOffsets);
}
 

import com.amazonaws.services.s3.model.S3ObjectSummary; //导入依赖的package包/类
public static List<S3ObjectSummary> listObjects(String bucket, String prefix, AmazonS3 s3) {
  List<S3ObjectSummary> objects = new ArrayList<>();
  ObjectListing listing;

  try {
    if (prefix == null) {
      listing = s3.listObjects(bucket);
    } else {
      listing = s3.listObjects(bucket, prefix);
    }

    objects.addAll(listing.getObjectSummaries());
    while (listing.isTruncated()) {
      listing = s3.listNextBatchOfObjects(listing);
      objects.addAll(listing.getObjectSummaries());
    }
  } catch (AmazonS3Exception e) {
   log.warn("listObjects for bucket '{}' prefix '{}' returned error code: {}", bucket, prefix, e.getStatusCode());
  }

  return objects;
}