Java SendMessageBatchRequest类代码示例

import com.amazonaws.services.sqs.model.SendMessageBatchRequest; //导入依赖的package包/类
@VisibleForTesting
static SendMessageBatchRequest createRequest(String queueUrl, Map<String, SendMessageEntry> entries) {
    return new SendMessageBatchRequest()
            .withQueueUrl(queueUrl)
            .withEntries(entries.entrySet().stream().map(keyValue -> {
                        SendMessageBatchRequestEntry entry = new SendMessageBatchRequestEntry()
                                .withId(keyValue.getKey())
                                .withMessageBody(keyValue.getValue().getBody());

                        keyValue.getValue().getDelay()
                                .ifPresent((delay) -> entry.setDelaySeconds((int) delay.getSeconds()));

                        return entry;
                    }).collect(Collectors.toList())
            );
}
 

import com.amazonaws.services.sqs.model.SendMessageBatchRequest; //导入依赖的package包/类
/**
 * Tests the {@link BatchSQSMsgSender#createBatches(org.apache.flume.Channel)} method. Tests happy path scenario.
 * <p>
 * <pre>
 * Inputs:
 *  channel = never empty
 *  batchSize = 5
 *  maxMessageSize = 10 Bytes
 *  each message size = 2 Bytes
 *
 * Expected Output:
 *  number of batches = 1
 *  number of messages in batch = 5
 * </pre>
 */
@Test
public void testCreateBatches() throws Exception {
    BatchSQSMsgSender sqsMsgSender =
        new BatchSQSMsgSender("https://some-fake/url", "us-east-1", "someAwsAccessKey", "someAwsSecretKey", 5, 10);

    byte[] mockMsgPayload = {'A', 'b'};
    Event mockEvent = Mockito.mock(Event.class);
    when(mockEvent.getBody()).thenReturn(mockMsgPayload);

    Channel mockChannel = Mockito.mock(Channel.class);
    when(mockChannel.take()).thenReturn(mockEvent);

    List<SendMessageBatchRequest> batches = sqsMsgSender.createBatches(mockChannel);

    Assert.assertNotNull(batches);
    Assert.assertEquals(1, batches.size());

    List<SendMessageBatchRequestEntry> msgEntries = batches.get(0).getEntries();
    Assert.assertNotNull(msgEntries);
    Assert.assertEquals(5, msgEntries.size());

    assertCorrectPayloadInEntries(mockMsgPayload, msgEntries);
}
 

import com.amazonaws.services.sqs.model.SendMessageBatchRequest; //导入依赖的package包/类
/**
 * Tests the {@link BatchSQSMsgSender#createBatches(org.apache.flume.Channel)} method. Tests the case when the
 * channel returns event with empty body.
 * <p>
 * <pre>
 * Inputs:
 *  channel = 1 event with empty body
 *  batchSize = 5
 *  maxMessageSize = 10 Bytes
 *
 * Expected Output:
 *  number of batches = 0
 * </pre>
 */
@Test
public void testCreateBatchesEventWithEmptyBody() throws Exception {
    BatchSQSMsgSender sqsMsgSender =
        new BatchSQSMsgSender("https://some-fake/url", "us-east-1", "someAwsAccessKey", "someAwsSecretKey", 5, 10);

    Channel mockChannel = Mockito.mock(Channel.class);
    Event mockEvent = Mockito.mock(Event.class);
    when(mockEvent.getBody()).thenReturn("".getBytes());
    when(mockChannel.take()).thenReturn(mockEvent);

    List<SendMessageBatchRequest> batches = sqsMsgSender.createBatches(mockChannel);

    Assert.assertNotNull(batches);
    Assert.assertEquals(0, batches.size());
}
 

import com.amazonaws.services.sqs.model.SendMessageBatchRequest; //导入依赖的package包/类
/**
 * Tests the {@link BatchSQSMsgSender#createBatches(org.apache.flume.Channel)} method. Tests the case when the
 * channel is empty after first event.
 * <p>
 * <pre>
 * Inputs:
 *  channel = 1 Event (Empty after first take)
 *  batchSize = 5
 *  maxMessageSize = 10 Bytes
 *
 * Expected Output:
 *  number of batches = 1
 *  number of messages in batch = 1
 * </pre>
 */
@Test
public void testCreateBatchesEmptyChannelAfterFirstTake() throws Exception {
    BatchSQSMsgSender sqsMsgSender =
        new BatchSQSMsgSender("https://some-fake/url", "us-east-1", "someAwsAccessKey", "someAwsSecretKey", 5, 10);
    byte[] mockMsgPayload = {'A', 'b'};
    Event mockEvent = Mockito.mock(Event.class);
    when(mockEvent.getBody()).thenReturn(mockMsgPayload);

    Channel mockChannel = Mockito.mock(Channel.class);
    when(mockChannel.take()).thenReturn(mockEvent).thenReturn(null);

    List<SendMessageBatchRequest> batches = sqsMsgSender.createBatches(mockChannel);

    Assert.assertNotNull(batches);
    Assert.assertEquals(1, batches.size());

    List<SendMessageBatchRequestEntry> msgEntries = batches.get(0).getEntries();
    Assert.assertNotNull(msgEntries);
    Assert.assertEquals(1, msgEntries.size());

    assertCorrectPayloadInEntries(mockMsgPayload, msgEntries);
}
 

import com.amazonaws.services.sqs.model.SendMessageBatchRequest; //导入依赖的package包/类
/**
 * Tests the {@link BatchSQSMsgSender#createBatches(org.apache.flume.Channel)} method. Tests the case when the
 * channel is empty after the last take for the batch.
 * <p>
 * <pre>
 * Inputs:
 *  channel = 5 Events (Empty after 5th take)
 *  batchSize = 5
 *  maxMessageSize = 10 Bytes
 *
 * Expected Output:
 *  number of batches = 1
 *  number of messages in batch = 5
 * </pre>
 */
@Test
public void testCreateBatchesEmptyChannelAfterLastTake() throws Exception {
    BatchSQSMsgSender sqsMsgSender =
        new BatchSQSMsgSender("https://some-fake/url", "us-east-1", "someAwsAccessKey", "someAwsSecretKey", 5, 10);
    byte[] mockMsgPayload = {'A', 'b'};
    Event mockEvent = Mockito.mock(Event.class);
    when(mockEvent.getBody()).thenReturn(mockMsgPayload);

    Channel mockChannel = Mockito.mock(Channel.class);
    when(mockChannel.take()).thenReturn(mockEvent, mockEvent, mockEvent, mockEvent, mockEvent, null);

    List<SendMessageBatchRequest> batches = sqsMsgSender.createBatches(mockChannel);

    Assert.assertNotNull(batches);
    Assert.assertEquals(1, batches.size());

    List<SendMessageBatchRequestEntry> msgEntries = batches.get(0).getEntries();
    Assert.assertNotNull(msgEntries);
    Assert.assertEquals(5, msgEntries.size());

    assertCorrectPayloadInEntries(mockMsgPayload, msgEntries);
}
 

import com.amazonaws.services.sqs.model.SendMessageBatchRequest; //导入依赖的package包/类
/**
 * Tests the {@link BatchSQSMsgSender#createBatches(org.apache.flume.Channel)} method. Tests the case when the
 * channel is empty in the middle of taking events for the batch
 * <p>
 * <pre>
 * Inputs:
 *  channel = 3 Events (Empty after 3rd take)
 *  batchSize = 5
 *  maxMessageSize = 10 Bytes
 *
 * Expected Output:
 *  number of batches = 1
 *  number of messages in batch = 3
 * </pre>
 */
@Test
public void testCreateBatchesEmptyChannelInTheMiddle() throws Exception {
    BatchSQSMsgSender sqsMsgSender =
        new BatchSQSMsgSender("https://some-fake/url", "us-east-1", "someAwsAccessKey", "someAwsSecretKey", 5, 10);
    byte[] mockMsgPayload = {'A', 'b'};
    Event mockEvent = Mockito.mock(Event.class);
    when(mockEvent.getBody()).thenReturn(mockMsgPayload);

    Channel mockChannel = Mockito.mock(Channel.class);
    when(mockChannel.take()).thenReturn(mockEvent, mockEvent, mockEvent, null);

    List<SendMessageBatchRequest> batches = sqsMsgSender.createBatches(mockChannel);

    Assert.assertNotNull(batches);
    Assert.assertEquals(1, batches.size());

    List<SendMessageBatchRequestEntry> msgEntries = batches.get(0).getEntries();
    Assert.assertNotNull(msgEntries);
    Assert.assertEquals(3, msgEntries.size());

    assertCorrectPayloadInEntries(mockMsgPayload, msgEntries);
}
 

import com.amazonaws.services.sqs.model.SendMessageBatchRequest; //导入依赖的package包/类
/**
 * Tests the {@link BatchSQSMsgSender#createBatches(org.apache.flume.Channel)} method. Tests the case when the
 * channel is not empty but contains events with empty body in the middle of taking events for the batch
 * <p>
 * <pre>
 * Inputs:
 *  channel = 4 Events (3 Events with Body and 4th Event empty)
 *  batchSize = 5
 *  maxMessageSize = 10 Bytes
 *
 * Expected Output:
 *  number of batches = 1
 *  number of messages in batch = 3
 * </pre>
 */
@Test
public void testCreateBatchesEmptyEventInTheMiddle() throws Exception {
    BatchSQSMsgSender sqsMsgSender =
        new BatchSQSMsgSender("https://some-fake/url", "us-east-1", "someAwsAccessKey", "someAwsSecretKey", 5, 10);
    byte[] mockMsgPayload = {'A', 'b'};
    byte[] mockEmptyMsgPayload = {};
    Event mockEvent = Mockito.mock(Event.class);
    Event mockEmptyEvent = Mockito.mock(Event.class);
    when(mockEvent.getBody()).thenReturn(mockMsgPayload);
    when(mockEmptyEvent.getBody()).thenReturn(mockEmptyMsgPayload);

    Channel mockChannel = Mockito.mock(Channel.class);
    when(mockChannel.take()).thenReturn(mockEvent, mockEvent, mockEvent, mockEmptyEvent);

    List<SendMessageBatchRequest> batches = sqsMsgSender.createBatches(mockChannel);

    Assert.assertNotNull(batches);
    Assert.assertEquals(1, batches.size());

    List<SendMessageBatchRequestEntry> msgEntries = batches.get(0).getEntries();
    Assert.assertNotNull(msgEntries);
    Assert.assertEquals(3, msgEntries.size());

    assertCorrectPayloadInEntries(mockMsgPayload, msgEntries);
}
 

import com.amazonaws.services.sqs.model.SendMessageBatchRequest; //导入依赖的package包/类
/**
 * Tests the {@link BatchSQSMsgSender#send(org.apache.flume.Channel)} method. Tests the happy path scenario.
 *
 * @throws Exception
 */
@Test
public void testSend() throws Exception {
    BatchSQSMsgSender sqsMsgSender =
        new BatchSQSMsgSender("https://some-fake/url", "us-east-1", "someAwsAccessKey", "someAwsSecretKey", 5, 10);
    AmazonSQS mockSqs = Mockito.mock(AmazonSQS.class);
    when(mockSqs.sendMessageBatch(any(SendMessageBatchRequest.class))).thenReturn(new SendMessageBatchResult());
    sqsMsgSender.setAmazonSQS(mockSqs);

    byte[] mockMsgPayload = {'A', 'b'};
    Event mockEvent = Mockito.mock(Event.class);
    when(mockEvent.getBody()).thenReturn(mockMsgPayload);

    Channel mockChannel = Mockito.mock(Channel.class);
    when(mockChannel.take()).thenReturn(mockEvent);

    sqsMsgSender.send(mockChannel);
}
 

import com.amazonaws.services.sqs.model.SendMessageBatchRequest; //导入依赖的package包/类
/**
 * Tests the {@link BatchSQSMsgSender#send(org.apache.flume.Channel)} method. Tests the failure scenario when AWS
 * SQS API throws AmazonServiceException.
 *
 * @throws Exception
 */
@Test(expected = EventDeliveryException.class)
public void testSendFailureAmazonServiceException() throws Exception {
    BatchSQSMsgSender sqsMsgSender =
        new BatchSQSMsgSender("https://some-fake/url", "us-east-1", "someAwsAccessKey", "someAwsSecretKey", 5, 10);
    AmazonSQS mockSqs = Mockito.mock(AmazonSQS.class);
    when(mockSqs.sendMessageBatch(any(SendMessageBatchRequest.class))).thenThrow(AmazonServiceException.class);
    sqsMsgSender.setAmazonSQS(mockSqs);

    byte[] mockMsgPayload = {'A', 'b'};
    Event mockEvent = Mockito.mock(Event.class);
    when(mockEvent.getBody()).thenReturn(mockMsgPayload);

    Channel mockChannel = Mockito.mock(Channel.class);
    when(mockChannel.take()).thenReturn(mockEvent);

    sqsMsgSender.send(mockChannel);
}
 

import com.amazonaws.services.sqs.model.SendMessageBatchRequest; //导入依赖的package包/类
/**
 * Tests the {@link BatchSQSMsgSender#send(org.apache.flume.Channel)} method. Tests the failure scenario when AWS
 * SQS API throws AmazonClientException.
 *
 * @throws Exception
 */
@Test(expected = EventDeliveryException.class)
public void testSendFailureAmazonClientException() throws Exception {
    BatchSQSMsgSender sqsMsgSender =
        new BatchSQSMsgSender("https://some-fake/url", "us-east-1", "someAwsAccessKey", "someAwsSecretKey", 5, 10);
    AmazonSQS mockSqs = Mockito.mock(AmazonSQS.class);
    when(mockSqs.sendMessageBatch(any(SendMessageBatchRequest.class))).thenThrow(AmazonClientException.class);
    sqsMsgSender.setAmazonSQS(mockSqs);

    byte[] mockMsgPayload = {'A', 'b'};
    Event mockEvent = Mockito.mock(Event.class);
    when(mockEvent.getBody()).thenReturn(mockMsgPayload);

    Channel mockChannel = Mockito.mock(Channel.class);
    when(mockChannel.take()).thenReturn(mockEvent);

    sqsMsgSender.send(mockChannel);
}
 

import com.amazonaws.services.sqs.model.SendMessageBatchRequest; //导入依赖的package包/类
@Test
public void shouldSendBatchesInSizeOfTen() throws Exception {
    when(mockAmazonSQS.sendMessageBatch(any(SendMessageBatchRequest.class))).thenReturn(mock(SendMessageBatchResult.class));
    ArgumentCaptor<SendMessageBatchRequest> captor = ArgumentCaptor.forClass(SendMessageBatchRequest.class);

    messagePublisher.postBatch(messageBatch(10), subject);
    verify(mockAmazonSQS, times(1)).sendMessageBatch(captor.capture());
    assertThat(captor.getValue().getEntries()).hasSize(10);

    messagePublisher.postBatch(messageBatch(20), subject);
    verify(mockAmazonSQS, times(3)).sendMessageBatch(any(SendMessageBatchRequest.class));

    messagePublisher.postBatch(messageBatch(11), subject);
    verify(mockAmazonSQS, times(5)).sendMessageBatch(captor.capture());
    assertThat(captor.getValue().getEntries()).hasSize(1);

    messagePublisher.postBatch(messageBatch(9), subject);
    verify(mockAmazonSQS, times(6)).sendMessageBatch(captor.capture());
    assertThat(captor.getValue().getEntries()).hasSize(9);
}
 

import com.amazonaws.services.sqs.model.SendMessageBatchRequest; //导入依赖的package包/类
void publishMessages(List<Message> messages) {
	logger.info("Sending {} messages", messages.size());
	SendMessageBatchRequest batch = new SendMessageBatchRequest(queueURL);
	messages.stream().forEach(msg -> {
		SendMessageBatchRequestEntry sendr = new SendMessageBatchRequestEntry(msg.getId(), msg.getPayload());
		batch.getEntries().add(sendr);
	});
	logger.info("sending {}", batch.getEntries().size());
	SendMessageBatchResult result = client.sendMessageBatch(batch);
	logger.info("send result {}", result.getFailed().toString());
}
 

import com.amazonaws.services.sqs.model.SendMessageBatchRequest; //导入依赖的package包/类
@Override
public int send(Channel channel) throws EventDeliveryException {
    int eventProcessedCounter = 0;
    // Create batch request
    List<SendMessageBatchRequest> batchRequests = createBatches(channel);

    for (SendMessageBatchRequest batchRequest : batchRequests) {
        // Send batch request
        SendMessageBatchResult result = null;
        try {
            result = this.amazonSQS.sendMessageBatch(batchRequest);
        }
        catch (AmazonServiceException ase) {
            // Throw request reached to SQS but the whole batch was rejected for some reason. Let the whole batch
            // be treated as "failed". Flume will retry the while batch
            throw new EventDeliveryException("Failure sending batch message request to Amazon SQS, " +
                "the request made it to SQS but was rejected for some reason.", ase);
        }
        catch (AmazonClientException ace) {
            throw new EventDeliveryException("Failure sending batch message request to Amazon SQS.", ace);
        }

        // Handle the result of the SQS batch request i.e., log errors, or fail the whole batch by throwing
        // EventDeliveryException in case of errors etc.
        handleResult(batchRequest, result);

        // The code reached here means there is nothing to rollback in this transaction. So increment the
        // eventProcessedCounter by the number of successfully sent messages.
        eventProcessedCounter += result.getSuccessful().size();
    }
    return eventProcessedCounter;
}
 

import com.amazonaws.services.sqs.model.SendMessageBatchRequest; //导入依赖的package包/类
/**
 * Handles SQS send message batch result and throws EventDeliveryException to cause the flume transaction to fail
 * and let flume retry the whole batch in case all the messages in the batch failed to be delivered to SQS.
 * Currently, this method does just logs errors and skips the messages in case some messages from the batched failed
 * to be delivered but some succeeded (i.e., partial batch failure).
 * <p>
 * TODO: Add retry logic instead letting flume drop the failed messages in case of partial batch failure
 *
 * @param batchRequest The SQS SendMessageBatchRequest
 * @param batchResult The SQS SendMessageBatchResult
 *
 * @throws EventDeliveryException In case all the messages in the batch failed to be delivered to SQS
 */
protected void handleResult(SendMessageBatchRequest batchRequest, SendMessageBatchResult batchResult)
    throws EventDeliveryException {

    List<SendMessageBatchRequestEntry> batchRequestEntries = batchRequest.getEntries();
    List<BatchResultErrorEntry> errors = batchResult.getFailed();

    int attemptedCount = batchRequestEntries == null ? 0 : batchRequestEntries.size();
    int errorCount = errors == null ? 0 : errors.size();

    if (errorCount > 0) {
        String errorMessage = buildErrorMessage(batchRequestEntries, errors);

        if (attemptedCount == errorCount) {
            // if it was a non-empty batch and if all the messages in the batch have errors then fail the whole
            // batch and let flume rollback the transaction and retry it
            // Just throw the EventDeliveryException. This will eventually cause the channel's transaction to
            // rollback.
            throw new EventDeliveryException(errorMessage);
        }
        else {
            // TODO: Add retry logic instead letting flume drop the failed messages in case of partial batch failure

            // Just log the error message and let flume drop failed messages in case of partial batch failures
            LOG.error(errorMessage);
        }
    }
}
 

import com.amazonaws.services.sqs.model.SendMessageBatchRequest; //导入依赖的package包/类
/**
 * Tests {@link BatchSQSMsgSender#createBatches(org.apache.flume.Channel)} method. Tests invalid characters not
 * allowed by the SQS. See [http://docs.aws.amazon
 * .com/AWSSimpleQueueService/latest/APIReference/API_SendMessageBatch.html]
 * for list of valid characters allowed by SQS.
 * <p>
 * <p>
 * <pre>
 * Inputs:
 *  channel = never empty. with messages containing invalid characters.
 *
 * Expected Output:
 *   The sink messages should not contain invalid characters
 * </pre>
 */
@Test
public void testInvalidCharacters() throws Exception {
    // See
    // http://stackoverflow.com/questions/16688523/aws-sqs-valid-characters
    // http://stackoverflow.com/questions/1169754/amazon-sqs-invalid-binary-character-in-message-body
    // https://forums.aws.amazon.com/thread.jspa?messageID=459090
    // http://stackoverflow.com/questions/16329695/invalid-binary-character-when-transmitting-protobuf-net
    // -messages-over-aws-sqs
    byte invalidCharByte = 0x1C;
    String mockMsg = "Test with some invalid chars at the end 0%2F>^F";
    byte[] origPayloadWithInvalidChars = ArrayUtils.add(mockMsg.getBytes(), invalidCharByte);

    BatchSQSMsgSender sqsMsgSender =
        new BatchSQSMsgSender("https://some-fake/url", "us-east-1", "someAwsAccessKey", "someAwsSecretKey", 1,
            origPayloadWithInvalidChars.length);

    Event mockEvent = Mockito.mock(Event.class);
    when(mockEvent.getBody()).thenReturn(origPayloadWithInvalidChars);

    Channel mockChannel = Mockito.mock(Channel.class);
    when(mockChannel.take()).thenReturn(mockEvent);

    List<SendMessageBatchRequest> batches = sqsMsgSender.createBatches(mockChannel);

    List<SendMessageBatchRequestEntry> msgEntries = batches.get(0).getEntries();
    assertCorrectPayloadInEntries(new String(origPayloadWithInvalidChars).trim().getBytes(), msgEntries);

    // Make sure that the message being sent by the sink doesn't contain the invalid characters
    for (SendMessageBatchRequestEntry entry : msgEntries) {
        Assert.assertNotNull(entry);
        Assert.assertTrue(ArrayUtils.contains(new String(origPayloadWithInvalidChars).getBytes(), invalidCharByte));
        Assert.assertTrue(!ArrayUtils.contains(entry.getMessageBody().getBytes(), invalidCharByte));
    }
}