Java PseudoTxnMemoryChannel类代码示例

import org.apache.flume.channel.PseudoTxnMemoryChannel; //导入依赖的package包/类
/**
 * Lack of exception test.
 */
@Test
public void testAppend() throws InterruptedException, LifecycleException,
    EventDeliveryException {

  Channel channel = new PseudoTxnMemoryChannel();
  Context context = new Context();
  Configurables.configure(channel, context);
  Configurables.configure(sink, context);

  sink.setChannel(channel);
  sink.start();

  for (int i = 0; i < 10; i++) {
    Event event = EventBuilder.withBody(("Test " + i).getBytes());
    channel.put(event);
    sink.process();
  }

  sink.stop();
}
 

import org.apache.flume.channel.PseudoTxnMemoryChannel; //导入依赖的package包/类
@Test
public void testAppendWithCustomSize() throws InterruptedException, LifecycleException,
        EventDeliveryException {

  Channel channel = new PseudoTxnMemoryChannel();
  Context context = new Context();
  context.put(LoggerSink.MAX_BYTES_DUMP_KEY, String.valueOf(30));
  Configurables.configure(channel, context);
  Configurables.configure(sink, context);

  sink.setChannel(channel);
  sink.start();

  for (int i = 0; i < 10; i++) {
    Event event = EventBuilder.withBody((Strings.padStart("Test " + i, 30, 'P')).getBytes());

    channel.put(event);
    sink.process();
  }

  sink.stop();
}
 

import org.apache.flume.channel.PseudoTxnMemoryChannel; //导入依赖的package包/类
@Test
public void testLifecycle() throws InterruptedException, LifecycleException {
    Channel channel = new PseudoTxnMemoryChannel();
    Context context = new Context();
    context.put(PentahoKettleSink.SINK_TRANS_PATH, getClass().getResource(SINK_TRANS_NAME).toExternalForm());
    context.put(PentahoKettleSink.SINK_EXECUTION_TYPE, "blocking");
    context.put(PentahoKettleSink.SINK_INJECTOR_NAME, "inject event");

    Configurables.configure(channel, new Context());
    Configurables.configure(sink, context);

    sink.setChannel(channel);
    sink.start();
    Assert.assertTrue("Reached start or error",
            LifecycleController.waitForOneOf(sink, LifecycleState.START_OR_ERROR));
    Assert.assertEquals("Sink is started", LifecycleState.START, sink.getLifecycleState());

    sink.stop();
    Assert.assertTrue("Reached stop or error", LifecycleController.waitForOneOf(sink, LifecycleState.STOP_OR_ERROR));
    Assert.assertEquals("Sink is stopped", LifecycleState.STOP, sink.getLifecycleState());
}
 

import org.apache.flume.channel.PseudoTxnMemoryChannel; //导入依赖的package包/类
/**
 * Lack of exception test.
 */
@Test
public void testBlockingSink() throws InterruptedException, LifecycleException, EventDeliveryException {

    Channel channel = new PseudoTxnMemoryChannel();
    Context context = new Context();
    context.put(PentahoKettleSink.SINK_TRANS_PATH, getClass().getResource(SINK_TRANS_NAME).toExternalForm());
    context.put(PentahoKettleSink.SINK_EXECUTION_TYPE, "blocking");
    context.put(PentahoKettleSink.SINK_INJECTOR_NAME, "inject event");

    Configurables.configure(channel, new Context());
    Configurables.configure(sink, context);

    sink.setChannel(channel);
    sink.start();

    for (int i = 0; i < 5; i++) {
        Event event = EventBuilder.withBody(("Test " + i).getBytes());

        channel.put(event);
        sink.process();
    }

    sink.stop();
}
 

import org.apache.flume.channel.PseudoTxnMemoryChannel; //导入依赖的package包/类
/**
 * Lack of exception test.
 */
@Test
public void testNonBlockingSink() throws InterruptedException, LifecycleException, EventDeliveryException {

    Channel channel = new PseudoTxnMemoryChannel();
    Context context = new Context();
    context.put(PentahoKettleSink.SINK_TRANS_PATH, getClass().getResource(SINK_TRANS_NAME).toExternalForm());
    context.put(PentahoKettleSink.SINK_EXECUTION_TYPE, "nonblocking");
    context.put(PentahoKettleSink.SINK_INJECTOR_NAME, "inject event");

    Configurables.configure(channel, new Context());
    Configurables.configure(sink, context);

    sink.setChannel(channel);
    sink.start();

    for (int i = 0; i < 5; i++) {
        Event event = EventBuilder.withBody(("Test " + i).getBytes());

        channel.put(event);
        sink.process();
    }

    sink.stop();
}
 

import org.apache.flume.channel.PseudoTxnMemoryChannel; //导入依赖的package包/类
@Test
public void testProcess() throws InterruptedException, LifecycleException,
    EventDeliveryException {

  Channel channel = new PseudoTxnMemoryChannel();
  Context context = new Context();

  context.put("logicalNode.name", "test");

  Configurables.configure(source, context);
  Configurables.configure(channel, context);

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
  source.start();

  for (long i = 0; i < 100; i++) {
    source.process();
    Event event = channel.take();

    Assert.assertArrayEquals(String.valueOf(i).getBytes(),
        new String(event.getBody()).getBytes());
  }
}
 

import org.apache.flume.channel.PseudoTxnMemoryChannel; //导入依赖的package包/类
@Test
public void testBatchProcessWithLifeCycle() throws InterruptedException, LifecycleException,
    EventDeliveryException {

  int batchSize = 10;

  Channel channel = new PseudoTxnMemoryChannel();
  Context context = new Context();

  context.put("logicalNode.name", "test");
  context.put("batchSize", Integer.toString(batchSize));

  Configurables.configure(source, context);
  Configurables.configure(channel, context);

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));

  source.start();

  for (long i = 0; i < 100; i++) {
    source.process();

    for (long j = batchSize; j > 0; j--) {
      Event event = channel.take();
      String expectedVal = String.valueOf(((i + 1) * batchSize) - j);
      String resultedVal = new String(event.getBody());
      Assert.assertTrue("Expected " + expectedVal + " is not equals to " +
          resultedVal, expectedVal.equals(resultedVal));
    }
  }

  source.stop();
}
 

import org.apache.flume.channel.PseudoTxnMemoryChannel; //导入依赖的package包/类
@Test
public void testLifecycle() throws InterruptedException,
    EventDeliveryException {

  Channel channel = new PseudoTxnMemoryChannel();
  Context context = new Context();

  context.put("logicalNode.name", "test");

  Configurables.configure(source, context);
  Configurables.configure(channel, context);

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));

  source.start();

  for (long i = 0; i < 100; i++) {
    source.process();
    Event event = channel.take();

    Assert.assertArrayEquals(String.valueOf(i).getBytes(),
        new String(event.getBody()).getBytes());
  }
  source.stop();
}
 

import org.apache.flume.channel.PseudoTxnMemoryChannel; //导入依赖的package包/类
@Test
public void testAppend() throws InterruptedException, LifecycleException,
    EventDeliveryException, IOException {

  Context context = new Context();

  context.put("sink.directory", tmpDir.getPath());
  context.put("sink.rollInterval", "1");
  context.put("sink.batchSize", "1");

  Configurables.configure(sink, context);

  Channel channel = new PseudoTxnMemoryChannel();
  Configurables.configure(channel, context);

  sink.setChannel(channel);
  sink.start();

  for (int i = 0; i < 10; i++) {
    Event event = new SimpleEvent();

    event.setBody(("Test event " + i).getBytes());

    channel.put(event);
    sink.process();

    Thread.sleep(500);
  }

  sink.stop();

  for (String file : sink.getDirectory().list()) {
    BufferedReader reader =
        new BufferedReader(new FileReader(new File(sink.getDirectory(), file)));

    String lastLine = null;
    String currentLine = null;

    while ((currentLine = reader.readLine()) != null) {
      lastLine = currentLine;
    }

    logger.debug("Produced file:{} lastLine:{}", file, lastLine);

    reader.close();
  }
}
 

import org.apache.flume.channel.PseudoTxnMemoryChannel; //导入依赖的package包/类
@Test
public void testAppend3()
    throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
  File tmpDir = new File("target/tmpLog");
  tmpDir.mkdirs();
  cleanDirectory(tmpDir);
  Context context = new Context();

  context.put("sink.directory", "target/tmpLog");
  context.put("sink.rollInterval", "0");
  context.put("sink.batchSize", "1");
  context.put("sink.pathManager.prefix", "test3-");
  context.put("sink.pathManager.extension", "txt");

  Configurables.configure(sink, context);

  Channel channel = new PseudoTxnMemoryChannel();
  Configurables.configure(channel, context);

  sink.setChannel(channel);
  sink.start();

  for (int i = 0; i < 10; i++) {
    Event event = new SimpleEvent();

    event.setBody(("Test event " + i).getBytes());

    channel.put(event);
    sink.process();

    Thread.sleep(500);
  }

  sink.stop();

  for (String file : sink.getDirectory().list()) {
    BufferedReader reader =
        new BufferedReader(new FileReader(new File(sink.getDirectory(), file)));

    String lastLine = null;
    String currentLine = null;

    while ((currentLine = reader.readLine()) != null) {
      lastLine = currentLine;
      logger.debug("Produced file:{} lastLine:{}", file, lastLine);
    }

    reader.close();
  }
}
 

import org.apache.flume.channel.PseudoTxnMemoryChannel; //导入依赖的package包/类
@Test
public void testRollTime()
    throws InterruptedException, LifecycleException, EventDeliveryException, IOException {
  File tmpDir = new File("target/tempLog");
  tmpDir.mkdirs();
  cleanDirectory(tmpDir);
  Context context = new Context();

  context.put("sink.directory", "target/tempLog/");
  context.put("sink.rollInterval", "1");
  context.put("sink.batchSize", "1");
  context.put("sink.pathManager", "rolltime");
  context.put("sink.pathManager.prefix", "test4-");
  context.put("sink.pathManager.extension", "txt");

  Configurables.configure(sink, context);

  Channel channel = new PseudoTxnMemoryChannel();
  Configurables.configure(channel, context);

  sink.setChannel(channel);
  sink.start();

  for (int i = 0; i < 10; i++) {
    Event event = new SimpleEvent();

    event.setBody(("Test event " + i).getBytes());

    channel.put(event);
    sink.process();

    Thread.sleep(500);
  }

  sink.stop();

  for (String file : sink.getDirectory().list()) {
    BufferedReader reader =
        new BufferedReader(new FileReader(new File(sink.getDirectory(), file)));

    String lastLine = null;
    String currentLine = null;

    while ((currentLine = reader.readLine()) != null) {
      lastLine = currentLine;
      logger.debug("Produced file:{} lastLine:{}", file, lastLine);
    }

    reader.close();
  }
}
 

import org.apache.flume.channel.PseudoTxnMemoryChannel; //导入依赖的package包/类
/**
 */
@Test
public void testIt() throws InterruptedException, LifecycleException, EventDeliveryException {
  // Starting Flume
  StormSink<SimpleConnectionParameters, SimpleEventSender> sink = new StormSink<SimpleConnectionParameters, SimpleEventSender>();

  Channel channel = new PseudoTxnMemoryChannel();
  Context context = new Context(stormSinkConfig.getMap());
  context.put("keep-alive", "1"); // Speeds up test
  Configurables.configure(channel, context);
  Configurables.configure(sink, context);

  sink.setChannel(channel);
  sink.start();

  // Checking location service registration
  List<SimpleServiceProvider> serviceProviders = simpleLocationService.getServiceProviders();
  assertThat(serviceProviders).hasSize(1);
  SimpleConnectionParameters connectionParams = serviceProviders.get(0).getConnectionParameters();

  // Queuing up some messages
  int nbEvents = 2 * 20; // Make it even and lower than channel capacity
  for (int i = 0; i < nbEvents; i++) {
    channel.put(EventBuilder.withBody(("Test queued" + i).getBytes()));
  }
  sink.process();

  SinkRunner sinkRunner = new SinkRunner();

  // Adding a couple of receptors
  SimpleEventReceptor eventReceptor1 = new SimpleEventReceptor(connectionParams);
  eventReceptor1.start();
  SimpleEventReceptor eventReceptor2 = new SimpleEventReceptor(connectionParams);
  eventReceptor2.start();
  EventReceptorTestUtils.waitConnected(eventReceptor1);
  EventReceptorTestUtils.waitConnected(eventReceptor2);

  // Sending queued events
  sink.process();
  assertThat(eventReceptor1.getEvents()).hasSize(nbEvents / 2);
  assertThat(eventReceptor2.getEvents()).hasSize(nbEvents / 2);

  // No event in channel
  sink.process();

  // Sending in real-time
  for (int j = 0; j < 5; j++) {
    for (int i = 0; i < nbEvents; i++) {
      channel.put(EventBuilder.withBody(("Test realtime" + i).getBytes()));
    }
    sink.process();
    assertThat(eventReceptor1.getEvents()).hasSize(nbEvents / 2);
    assertThat(eventReceptor2.getEvents()).hasSize(nbEvents / 2);
  }

  // Terminating test
  eventReceptor1.stop();
  eventReceptor2.stop();
  EventReceptorTestUtils.waitDisconnected(eventReceptor1);
  EventReceptorTestUtils.waitDisconnected(eventReceptor2);

  sink.stop();
}
 

import org.apache.flume.channel.PseudoTxnMemoryChannel; //导入依赖的package包/类
@Test
public void testAppend2() throws InterruptedException, LifecycleException,
    EventDeliveryException, IOException {

  Context context = new Context();

  context.put("sink.directory", tmpDir.getPath());
  context.put("sink.rollInterval", "0");
  context.put("sink.batchSize", "1");


  Configurables.configure(sink, context);

  Channel channel = new PseudoTxnMemoryChannel();
  Configurables.configure(channel, context);

  sink.setChannel(channel);
  sink.start();

  for (int i = 0; i < 10; i++) {
    Event event = new SimpleEvent();

    event.setBody(("Test event " + i).getBytes());

    channel.put(event);
    sink.process();

    Thread.sleep(500);
  }

  sink.stop();

  for (String file : sink.getDirectory().list()) {
    BufferedReader reader =
        new BufferedReader(new FileReader(new File(sink.getDirectory(), file)));

    String lastLine = null;
    String currentLine = null;

    while ((currentLine = reader.readLine()) != null) {
      lastLine = currentLine;
      logger.debug("Produced file:{} lastLine:{}", file, lastLine);
    }


    reader.close();
  }
}