Java S3EventNotification.S3EventNotificationRecord方法代码示例

import com.amazonaws.services.s3.event.S3EventNotification; //导入方法依赖的package包/类
@Override
public Void handleRequest(S3Event s3Event, Context context) {

    Collection<Partition> requiredPartitions = new HashSet<>();
    TableService tableService = new TableService();

    for (S3EventNotification.S3EventNotificationRecord record : s3Event.getRecords()) {

        String bucket = record.getS3().getBucket().getName();
        String key = record.getS3().getObject().getKey();

        System.out.printf("S3 event [Event: %s, Bucket: %s, Key: %s]%n", record.getEventName(), bucket, key);

        S3Object s3Object = new S3Object(bucket, key);

        if (s3Object.hasDateTimeKey()) {
            requiredPartitions.add(partitionConfig.createPartitionFor(s3Object));
        }
    }

    if (!requiredPartitions.isEmpty()) {
        Collection<Partition> missingPartitions = determineMissingPartitions(
                partitionConfig.tableName(),
                requiredPartitions,
                tableService);
        tableService.addPartitions(partitionConfig.tableName(), missingPartitions);
    }

    return null;
}
 

import com.amazonaws.services.s3.event.S3EventNotification; //导入方法依赖的package包/类
@Override
public Void handleRequest(S3Event s3Event, Context context) {

    Collection<Partition> partitionsToRemove = new HashSet<>();
    TableService tableService = new TableService();

    for (S3EventNotification.S3EventNotificationRecord record : s3Event.getRecords()) {

        String bucket = record.getS3().getBucket().getName();
        String key = record.getS3().getObject().getKey();

        System.out.printf("S3 event [Event: %s, Bucket: %s, Key: %s]%n", record.getEventName(), bucket, key);

        S3Object s3Object = new S3Object(bucket, key);

        if (s3Object.hasDateTimeKey()) {
            partitionsToRemove.add(partitionConfig.createPartitionFor(s3Object));
        }
    }

    if (!partitionsToRemove.isEmpty()) {
        tableService.removePartitions(
                partitionConfig.tableName(),
                partitionsToRemove.stream().map(Partition::spec).collect(Collectors.toList()));
    }

    return null;
}
 

import com.amazonaws.services.s3.event.S3EventNotification; //导入方法依赖的package包/类
@Override
public Void handleRequest(S3Event s3Event, Context context){

    Collection<Partition>requiredPartitions = new HashSet<>();
    TableService tableService = new TableService();
    DynamoDB dynamoDBClient=new DynamoDB(new AmazonDynamoDBClient(new EnvironmentVariableCredentialsProvider()));

    for(S3EventNotification.S3EventNotificationRecord record:s3Event.getRecords()){

        String bucket=record.getS3().getBucket().getName();
        String key=record.getS3().getObject().getKey();

        System.out.printf("S3event[Event:%s,Bucket:%s,Key:%s]%n",record.getEventName(),bucket,key);

        S3Object s3Object=new S3Object(bucket,key);

        if(s3Object.hasDateTimeKey()){
            Partition partition = partitionConfig.createPartitionFor(s3Object);

            //Check if the partition exists in DynamoDBtable, if not add the partition details to the table, skip otherwise
            if (tryAddMissingPartition(partitionConfig.dynamoDBTableName(), dynamoDBClient, partition)) {
                requiredPartitions.add(partition);
            }
        }
    }

    if(!requiredPartitions.isEmpty()){
        tableService.addPartitions(partitionConfig.tableName(),requiredPartitions, true);
    }

    return null;
}
 

import com.amazonaws.services.s3.event.S3EventNotification; //导入方法依赖的package包/类
@Before
public void before() {
    initMocks(this);
    handler = new CloudFrontLogEventHandler();

    String arn = System.getProperty("arn");
    String bucketName = System.getProperty("bucketName");
    String logKey = System.getProperty("logKey");

    Preconditions.checkNotNull(arn, "You must pass the arn for this lambda to run in a mocked manner, the arn is used to get the env name ex: arn:aws:lambda:us-west-2:1111111:function:dev-gateway-db2599d1-6d86-LambdaWAFBlacklistingFun-1LSORI5GUP95H");
    Preconditions.checkNotNull(bucketName, "You must supply a bucket for the lambda to read a log from");
    Preconditions.checkNotNull(logKey, "You must supply a key to a log for the lambda to read");

    List<S3EventNotification.S3EventNotificationRecord> records = Lists.newArrayList();
    S3EventNotification.S3EventNotificationRecord record = mock(S3EventNotification.S3EventNotificationRecord.class);
    records.add(record);

    when(event.getRecords()).thenReturn(records);

    S3EventNotification.S3Entity s3Entity = mock(S3EventNotification.S3Entity.class);
    S3EventNotification.S3BucketEntity bucketEntity = mock(S3EventNotification.S3BucketEntity.class);
    S3EventNotification.S3ObjectEntity objectEntity = mock(S3EventNotification.S3ObjectEntity.class);
    when(s3Entity.getBucket()).thenReturn(bucketEntity);
    when(s3Entity.getObject()).thenReturn(objectEntity);

    when(record.getS3()).thenReturn(s3Entity);

    when(context.getInvokedFunctionArn()).thenReturn(arn);
    when(bucketEntity.getName()).thenReturn(bucketName);
    when(objectEntity.getKey()).thenReturn(logKey);
}
 

import com.amazonaws.services.s3.event.S3EventNotification; //导入方法依赖的package包/类
@Test
public void testThatHandleEventCallsProcessEventsOnTheProcessors() throws IOException {
    String bucketName = "bucketname";
    String arn = "foo";

    Processor processor = mock(Processor.class);
    List<Processor> processors = Lists.newLinkedList();
    processors.add(processor);

    handler.overrideProcessors(processors);
    CloudFrontLogHandlerConfig params = new CloudFrontLogHandlerConfig();
    doReturn(params).when(handler).getConfiguration(arn);

    Context context = mock(Context.class);
    when(context.getInvokedFunctionArn()).thenReturn(arn);

    S3Event event = mock(S3Event.class);
    List<S3EventNotification.S3EventNotificationRecord> records = Lists.newArrayList();
    S3EventNotification.S3EventNotificationRecord record = mock(S3EventNotification.S3EventNotificationRecord.class);
    records.add(record);
    when(event.getRecords()).thenReturn(records);
    S3EventNotification.S3Entity s3Entity = mock(S3EventNotification.S3Entity.class);
    S3EventNotification.S3BucketEntity bucketEntity = mock(S3EventNotification.S3BucketEntity.class);
    S3EventNotification.S3ObjectEntity objectEntity = mock(S3EventNotification.S3ObjectEntity.class);
    when(s3Entity.getBucket()).thenReturn(bucketEntity);
    when(s3Entity.getObject()).thenReturn(objectEntity);
    when(record.getS3()).thenReturn(s3Entity);
    when(bucketEntity.getName()).thenReturn(bucketName);
    when(objectEntity.getKey()).thenReturn("access.log.gz");
    when(amazonS3Client.getObject(isA(GetObjectRequest.class))).thenReturn(mock(S3Object.class));
    doReturn(null).when(handler).ingestLogStream(null);

    handler.handleNewS3Event(event, context);

    verify(processor, times(1)).processLogEvents(null, params, bucketName);
}
 

import com.amazonaws.services.s3.event.S3EventNotification; //导入方法依赖的package包/类
@Test
public void testThatHandleEventCallsDoesNotProcessEventsOnTheProcessorsWhenNotALogFile() throws IOException {
    String bucketName = "bucketname";
    String arn = "foo";

    Processor processor = mock(Processor.class);
    List<Processor> processors = Lists.newLinkedList();
    processors.add(processor);

    handler.overrideProcessors(processors);
    CloudFrontLogHandlerConfig params = new CloudFrontLogHandlerConfig();
    doReturn(params).when(handler).getConfiguration(arn);

    Context context = mock(Context.class);
    when(context.getInvokedFunctionArn()).thenReturn(arn);

    S3Event event = mock(S3Event.class);
    List<S3EventNotification.S3EventNotificationRecord> records = Lists.newArrayList();
    S3EventNotification.S3EventNotificationRecord record = mock(S3EventNotification.S3EventNotificationRecord.class);
    records.add(record);
    when(event.getRecords()).thenReturn(records);
    S3EventNotification.S3Entity s3Entity = mock(S3EventNotification.S3Entity.class);
    S3EventNotification.S3BucketEntity bucketEntity = mock(S3EventNotification.S3BucketEntity.class);
    S3EventNotification.S3ObjectEntity objectEntity = mock(S3EventNotification.S3ObjectEntity.class);
    when(s3Entity.getBucket()).thenReturn(bucketEntity);
    when(s3Entity.getObject()).thenReturn(objectEntity);
    when(record.getS3()).thenReturn(s3Entity);
    when(bucketEntity.getName()).thenReturn(bucketName);
    when(objectEntity.getKey()).thenReturn("data.json");
    when(amazonS3Client.getObject(isA(GetObjectRequest.class))).thenReturn(mock(S3Object.class));
    doReturn(null).when(handler).ingestLogStream(null);

    handler.handleNewS3Event(event, context);

    verify(processor, times(0)).processLogEvents(null, params, bucketName);
}
 

import com.amazonaws.services.s3.event.S3EventNotification; //导入方法依赖的package包/类
@Test
public void testThatHandleEventDoesNotExplodeWhenTheFirstProcessorErrorsOut() throws IOException {
    String bucketName = "bucketname";
    String arn = "foo";

    Processor processor = mock(Processor.class);
    Processor processor2 = mock(Processor.class);
    List<Processor> processors = Lists.newLinkedList();
    processors.add(processor);
    doThrow(new RuntimeException("foo")).when(processor).processLogEvents(any(), any(), any());
    processors.add(processor2);

    handler.overrideProcessors(processors);
    CloudFrontLogHandlerConfig params = new CloudFrontLogHandlerConfig();
    doReturn(params).when(handler).getConfiguration(arn);

    Context context = mock(Context.class);
    when(context.getInvokedFunctionArn()).thenReturn(arn);

    S3Event event = mock(S3Event.class);
    List<S3EventNotification.S3EventNotificationRecord> records = Lists.newArrayList();
    S3EventNotification.S3EventNotificationRecord record = mock(S3EventNotification.S3EventNotificationRecord.class);
    records.add(record);
    when(event.getRecords()).thenReturn(records);
    S3EventNotification.S3Entity s3Entity = mock(S3EventNotification.S3Entity.class);
    S3EventNotification.S3BucketEntity bucketEntity = mock(S3EventNotification.S3BucketEntity.class);
    S3EventNotification.S3ObjectEntity objectEntity = mock(S3EventNotification.S3ObjectEntity.class);
    when(s3Entity.getBucket()).thenReturn(bucketEntity);
    when(s3Entity.getObject()).thenReturn(objectEntity);
    when(record.getS3()).thenReturn(s3Entity);
    when(bucketEntity.getName()).thenReturn(bucketName);
    when(objectEntity.getKey()).thenReturn("access.log.gz");
    when(amazonS3Client.getObject(isA(GetObjectRequest.class))).thenReturn(mock(S3Object.class));
    doReturn(null).when(handler).ingestLogStream(null);

    handler.handleNewS3Event(event, context);

    verify(processor, times(1)).processLogEvents(null, params, bucketName);
    verify(processor2, times(1)).processLogEvents(null, params, bucketName);
}
 

import com.amazonaws.services.s3.event.S3EventNotification; //导入方法依赖的package包/类
protected JsonEvent toJson(S3EventNotification.S3EventNotificationRecord record) {
   return new JsonEvent()
        .put("bucket_name", record.getS3().getBucket().getName())
            .put("bucket_arn", record.getS3().getBucket().getArn())
            .put("key", record.getS3().getObject().getKey())
            .put("etag", record.getS3().getObject().geteTag())
            .put("size", record.getS3().getObject().getSizeAsLong());
}
 

import com.amazonaws.services.s3.event.S3EventNotification; //导入方法依赖的package包/类
/**
 * The handler that will get triggered by the CloudFront adding a new log chunk into the CloudFront Log S3 Bucket.
 * Streams the log from S3 and processes each line, which represents a request to Cerberus.
 * http://docs.aws.amazon.com/AmazonCloudFront/latest/DeveloperGuide/AccessLogs.html#LogFileFormat
 *
 * @param context, the context of the lambda fn
 */
public void handleNewS3Event(S3Event event, Context context) throws IOException {
    CloudFrontLogHandlerConfig config =
            getConfiguration(context.getInvokedFunctionArn());

    log.info(String.format("Found CloudFormation stack and derived params: %s",
            objectMapper.writeValueAsString(config)));

    for (S3EventNotification.S3EventNotificationRecord s3EventNotificationRecord : event.getRecords()){
        String bucketName = s3EventNotificationRecord.getS3().getBucket().getName();
        String key = s3EventNotificationRecord.getS3().getObject().getKey();

        // Only process the log files from CF they end in .gz
        if (! key.endsWith(".gz")) {
            return;
        }

        log.info(String.format("Triggered from %s/%s", bucketName, key));
        S3Object logObject  = amazonS3Client.getObject(new GetObjectRequest(bucketName, key));
        List<CloudFrontLogEvent> logEvents = ingestLogStream(logObject.getObjectContent());

        logEventProcessors.forEach(processor -> {
            try {
                processor.processLogEvents(logEvents, config, bucketName);
            } catch (Throwable t) {
                log.error(String.format("Failed to run log processor %s", processor.getClass()), t);

                // Send a message to slack if its configured to do so
                if (StringUtils.isNotBlank(config.getSlackWebHookUrl())) {
                    String text = String.format("Failed to run log processor %s, env: %s reason: %s",
                            processor.getClass(), config.getEnv(), t.getMessage());
                    Message.Builder builder = new Message.Builder(text).userName("Cloud-Front-Event-Handler");

                    if (StringUtils.startsWith(config.getSlackIcon(), "http")) {
                        builder.iconUrl(config.getSlackIcon());
                    } else {
                        builder.iconEmoji(config.getSlackIcon());
                    }

                    new SlackClient(config.getSlackWebHookUrl()).sendMessage(builder.build());
                }
            }
        });
    }
}
 

import com.amazonaws.services.s3.event.S3EventNotification; //导入方法依赖的package包/类
/**
 * Create a new instance of S3Event
 * @param records A list of S3 event notification records
 */
public S3Event(List<S3EventNotification.S3EventNotificationRecord> records) {
    super(records);
}