Java Sink.Status方法代码示例

import org.apache.flume.Sink; //导入方法依赖的package包/类
private Sink.Status prepareAndSend(Context context, String msg)
    throws EventDeliveryException {
  Sink kafkaSink = new KafkaSink();
  Configurables.configure(kafkaSink, context);
  Channel memoryChannel = new MemoryChannel();
  Configurables.configure(memoryChannel, context);
  kafkaSink.setChannel(memoryChannel);
  kafkaSink.start();

  Transaction tx = memoryChannel.getTransaction();
  tx.begin();
  Event event = EventBuilder.withBody(msg.getBytes());
  memoryChannel.put(event);
  tx.commit();
  tx.close();

  return kafkaSink.process();
}
 

import org.apache.flume.Sink; //导入方法依赖的package包/类
@Test
public void testStaticTopic() {
  Context context = prepareDefaultContext();
  // add the static topic
  context.put(TOPIC_CONFIG, TestConstants.STATIC_TOPIC);
  String msg = "static-topic-test";

  try {
    Sink.Status status = prepareAndSend(context, msg);
    if (status == Sink.Status.BACKOFF) {
      fail("Error Occurred");
    }
  } catch (EventDeliveryException ex) {
    // ignore
  }

  String fetchedMsg = new String((byte[]) testUtil.getNextMessageFromConsumer(
      TestConstants.STATIC_TOPIC).message());
  assertEquals(msg, fetchedMsg);
}
 

import org.apache.flume.Sink; //导入方法依赖的package包/类
@Test
public void testEmptyChannel() throws UnsupportedEncodingException, EventDeliveryException {
  Sink kafkaSink = new KafkaSink();
  Context context = prepareDefaultContext();
  Configurables.configure(kafkaSink, context);
  Channel memoryChannel = new MemoryChannel();
  Configurables.configure(memoryChannel, context);
  kafkaSink.setChannel(memoryChannel);
  kafkaSink.start();

  Sink.Status status = kafkaSink.process();
  if (status != Sink.Status.BACKOFF) {
    fail("Error Occurred");
  }
  assertNull(testUtil.getNextMessageFromConsumer(DEFAULT_TOPIC));
}
 

import org.apache.flume.Sink; //导入方法依赖的package包/类
@Test
public void testProcess() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();

  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createServer(new MockAvroServer());

  server.start();

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
 

import org.apache.flume.Sink; //导入方法依赖的package包/类
@Test
public void testProcess() throws Exception {

  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  src = new ThriftTestingSource(ThriftTestingSource.HandlerType.OK.name(),
    port, ThriftRpcClient.COMPACT_PROTOCOL);

  channel.start();
  sink.start();

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 11; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();
  for (int i = 0; i < 6; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertEquals(11, src.flumeEvents.size());
  Assert.assertEquals(6, src.batchCount);
  Assert.assertEquals(0, src.individualCount);

}
 

import org.apache.flume.Sink; //导入方法依赖的package包/类
@Test
public void testSslProcess() throws Exception {
  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  src = new ThriftTestingSource(ThriftTestingSource.HandlerType.OK.name(), port,
          ThriftRpcClient.COMPACT_PROTOCOL, "src/test/resources/keystorefile.jks",
          "password", KeyManagerFactory.getDefaultAlgorithm(), "JKS");
  Context context = new Context();
  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("ssl", String.valueOf(true));
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));
  context.put("truststore", "src/test/resources/truststorefile.jks");
  context.put("truststore-password", "password");
  context.put("trustmanager-type", TrustManagerFactory.getDefaultAlgorithm());

  Configurables.configure(sink, context);
  channel.start();
  sink.start();
  Transaction transaction = channel.getTransaction();
  transaction.begin();
  for (int i = 0; i < 11; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();
  for (int i = 0; i < 6; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }
  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertEquals(11, src.flumeEvents.size());
  Assert.assertEquals(6, src.batchCount);
  Assert.assertEquals(0, src.individualCount);
}
 

import org.apache.flume.Sink; //导入方法依赖的package包/类
@Test
public void testCustomSelector() throws Exception {
  Channel ch = new MockChannel();
  int n = 10;
  int numEvents = n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  // s1 always fails
  s1.setFail(true);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);

  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  // This selector will result in all events going to s2
  Context ctx = getContext(FixedOrderSelector.class.getCanonicalName());
  ctx.put("selector." + FixedOrderSelector.SET_ME, "foo");
  LoadBalancingSinkProcessor lbsp = getProcessor(sinks, ctx);

  Sink.Status s = Sink.Status.READY;
  while (s != Sink.Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertTrue(s1.getEvents().size() == 0);
  Assert.assertTrue(s2.getEvents().size() == n);
  Assert.assertTrue(s3.getEvents().size() == 0);
}
 

import org.apache.flume.Sink; //导入方法依赖的package包/类
@Test
public void testIRCSinkMissingSplitLineProperty() {
  Sink ircSink = new IRCSink();
  ircSink.setName("IRC Sink - " + UUID.randomUUID().toString());
  Context context = new Context();
  context.put("hostname", "localhost");
  context.put("port", String.valueOf(ircServerPort));
  context.put("nick", "flume");
  context.put("password", "flume");
  context.put("user", "flume");
  context.put("name", "flume-dev");
  context.put("chan", "flume");
  context.put("splitchars", "false");
  Configurables.configure(ircSink, context);
  Channel memoryChannel = new MemoryChannel();
  Configurables.configure(memoryChannel, context);
  ircSink.setChannel(memoryChannel);
  ircSink.start();
  Transaction txn = memoryChannel.getTransaction();
  txn.begin();
  Event event = EventBuilder.withBody("Dummy Event".getBytes());
  memoryChannel.put(event);
  txn.commit();
  txn.close();
  try {
    Sink.Status status = ircSink.process();
    if (status == Sink.Status.BACKOFF) {
      fail("Error occured");
    }
  } catch (EventDeliveryException eDelExcp) {
    // noop
  }
}
 

import org.apache.flume.Sink; //导入方法依赖的package包/类
@Test
public void testDefaultTopic() {
  Sink kafkaSink = new KafkaSink();
  Context context = prepareDefaultContext();
  Configurables.configure(kafkaSink, context);
  Channel memoryChannel = new MemoryChannel();
  Configurables.configure(memoryChannel, context);
  kafkaSink.setChannel(memoryChannel);
  kafkaSink.start();

  String msg = "default-topic-test";
  Transaction tx = memoryChannel.getTransaction();
  tx.begin();
  Event event = EventBuilder.withBody(msg.getBytes());
  memoryChannel.put(event);
  tx.commit();
  tx.close();

  try {
    Sink.Status status = kafkaSink.process();
    if (status == Sink.Status.BACKOFF) {
      fail("Error Occurred");
    }
  } catch (EventDeliveryException ex) {
    // ignore
  }

  String fetchedMsg = new String((byte[]) testUtil.getNextMessageFromConsumer(DEFAULT_TOPIC)
                                                  .message());
  assertEquals(msg, fetchedMsg);
}
 

import org.apache.flume.Sink; //导入方法依赖的package包/类
@Test
public void testTopicAndKeyFromHeader() throws UnsupportedEncodingException {
  Sink kafkaSink = new KafkaSink();
  Context context = prepareDefaultContext();
  Configurables.configure(kafkaSink, context);
  Channel memoryChannel = new MemoryChannel();
  Configurables.configure(memoryChannel, context);
  kafkaSink.setChannel(memoryChannel);
  kafkaSink.start();

  String msg = "test-topic-and-key-from-header";
  Map<String, String> headers = new HashMap<String, String>();
  headers.put("topic", TestConstants.CUSTOM_TOPIC);
  headers.put("key", TestConstants.CUSTOM_KEY);
  Transaction tx = memoryChannel.getTransaction();
  tx.begin();
  Event event = EventBuilder.withBody(msg.getBytes(), headers);
  memoryChannel.put(event);
  tx.commit();
  tx.close();

  try {
    Sink.Status status = kafkaSink.process();
    if (status == Sink.Status.BACKOFF) {
      fail("Error Occurred");
    }
  } catch (EventDeliveryException ex) {
    // ignore
  }

  MessageAndMetadata fetchedMsg =
      testUtil.getNextMessageFromConsumer(TestConstants.CUSTOM_TOPIC);

  assertEquals(msg, new String((byte[]) fetchedMsg.message(), "UTF-8"));
  assertEquals(TestConstants.CUSTOM_KEY,
               new String((byte[]) fetchedMsg.key(), "UTF-8"));
}
 

import org.apache.flume.Sink; //导入方法依赖的package包/类
@Test
public void testFailedConnect() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {

  setUp();
  Event event = EventBuilder.withBody("test event 1",
      Charset.forName("UTF8"));
  Server server = createServer(new MockAvroServer());

  server.start();
  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Thread.sleep(500L); // let socket startup
  server.close();
  Thread.sleep(500L); // sleep a little to allow close occur

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    boolean threwException = false;
    try {
      sink.process();
    } catch (EventDeliveryException e) {
      threwException = true;
    }
    Assert.assertTrue("Must throw EventDeliveryException if disconnected",
        threwException);
  }

  server = createServer(new MockAvroServer());
  server.start();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));
  server.close();
}
 

import org.apache.flume.Sink; //导入方法依赖的package包/类
@Test
public void testSslProcess() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();
  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createSslServer(new MockAvroServer());

  server.start();

  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("ssl", String.valueOf(true));
  context.put("trust-all-certs", String.valueOf(true));
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));

  Configurables.configure(sink, context);

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
 

import org.apache.flume.Sink; //导入方法依赖的package包/类
@Test
public void testSslProcessWithTrustStore() throws InterruptedException,
    EventDeliveryException, InstantiationException, IllegalAccessException {
  setUp();
  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  Server server = createSslServer(new MockAvroServer());

  server.start();

  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("ssl", String.valueOf(true));
  context.put("truststore", "src/test/resources/truststore.jks");
  context.put("truststore-password", "password");
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));

  Configurables.configure(sink, context);

  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.START_OR_ERROR, 5000));

  Transaction transaction = channel.getTransaction();

  transaction.begin();
  for (int i = 0; i < 10; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  for (int i = 0; i < 5; i++) {
    Sink.Status status = sink.process();
    Assert.assertEquals(Sink.Status.READY, status);
  }

  Assert.assertEquals(Sink.Status.BACKOFF, sink.process());

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
      LifecycleState.STOP_OR_ERROR, 5000));

  server.close();
}
 

import org.apache.flume.Sink; //导入方法依赖的package包/类
@Test
public void testSslSinkWithNonSslServer() throws Exception {
  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  src = new ThriftTestingSource(ThriftTestingSource.HandlerType.OK.name(),
          port, ThriftRpcClient.COMPACT_PROTOCOL);

  Context context = new Context();
  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("ssl", String.valueOf(true));
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));
  context.put("truststore", "src/test/resources/truststorefile.jks");
  context.put("truststore-password", "password");
  context.put("trustmanager-type", TrustManagerFactory.getDefaultAlgorithm());

  Configurables.configure(sink, context);
  channel.start();
  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
          LifecycleState.START_OR_ERROR, 5000));
  Transaction transaction = channel.getTransaction();
  transaction.begin();
  for (int i = 0; i < 11; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  boolean failed = false;
  try {
    for (int i = 0; i < 6; i++) {
      Sink.Status status = sink.process();
      failed = true;
    }
  } catch (EventDeliveryException ex) {
    // This is correct
  }

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
          LifecycleState.STOP_OR_ERROR, 5000));
  if (failed) {
    Assert.fail("SSL-enabled sink successfully connected to a non-SSL-enabled server, " +
                "that's wrong.");
  }
}
 

import org.apache.flume.Sink; //导入方法依赖的package包/类
@Test
public void testSslSinkWithNonTrustedCert() throws Exception {
  Event event = EventBuilder.withBody("test event 1", Charsets.UTF_8);
  src = new ThriftTestingSource(ThriftTestingSource.HandlerType.OK.name(), port,
          ThriftRpcClient.COMPACT_PROTOCOL, "src/test/resources/keystorefile.jks",
          "password", KeyManagerFactory.getDefaultAlgorithm(), "JKS");

  Context context = new Context();
  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("ssl", String.valueOf(true));
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));

  Configurables.configure(sink, context);
  channel.start();
  sink.start();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
          LifecycleState.START_OR_ERROR, 5000));
  Transaction transaction = channel.getTransaction();
  transaction.begin();
  for (int i = 0; i < 11; i++) {
    channel.put(event);
  }
  transaction.commit();
  transaction.close();

  boolean failed = false;
  try {
    for (int i = 0; i < 6; i++) {
      Sink.Status status = sink.process();
      failed = true;
    }
  } catch (EventDeliveryException ex) {
    // This is correct
  }

  sink.stop();
  Assert.assertTrue(LifecycleController.waitForOneOf(sink,
          LifecycleState.STOP_OR_ERROR, 5000));
  if (failed) {
    Assert.fail("SSL-enabled sink successfully connected to a server with an " +
                "untrusted certificate when it should have failed");
  }
}