Java MemoryChannel类代码示例

import org.apache.flume.channel.MemoryChannel; //导入依赖的package包/类
public void setUp(String compressionType, int compressionLevel) {
  if (sink != null) {
    throw new RuntimeException("double setup");
  }
  sink = new AvroSink();
  channel = new MemoryChannel();

  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("batch-size", String.valueOf(2));
  context.put("connect-timeout", String.valueOf(2000L));
  context.put("request-timeout", String.valueOf(3000L));
  if (compressionType.equals("deflate")) {
    context.put("compression-type", compressionType);
    context.put("compression-level", Integer.toString(compressionLevel));
  }

  sink.setChannel(channel);

  Configurables.configure(sink, context);
  Configurables.configure(channel, context);
}
 

import org.apache.flume.channel.MemoryChannel; //导入依赖的package包/类
@Before
public void setUp() throws Exception {
  sink = new ThriftSink();
  channel = new MemoryChannel();
  hostname = "0.0.0.0";
  port = random.nextInt(50000) + 1024;
  Context context = new Context();

  context.put("hostname", hostname);
  context.put("port", String.valueOf(port));
  context.put("batch-size", String.valueOf(2));
  context.put("request-timeout", String.valueOf(2000L));
  context.put(ThriftRpcClient.CONFIG_PROTOCOL, ThriftRpcClient.COMPACT_PROTOCOL);
  sink.setChannel(channel);

  Configurables.configure(sink, context);
  Configurables.configure(channel, context);
}
 

import org.apache.flume.channel.MemoryChannel; //导入依赖的package包/类
@Before
public void setUp() throws UnknownHostException {
  localhost = InetAddress.getByName("127.0.0.1");
  source = new AvroSource();
  channel = new MemoryChannel();

  Configurables.configure(channel, new Context());

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
}
 

import org.apache.flume.channel.MemoryChannel; //导入依赖的package包/类
private void init(String keepFields) {
  source = new SyslogUDPSource();
  channel = new MemoryChannel();

  Configurables.configure(channel, new Context());

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
  Context context = new Context();
  context.put("host", InetAddress.getLoopbackAddress().getHostAddress());
  context.put("port", String.valueOf(TEST_SYSLOG_PORT));
  context.put("keepFields", keepFields);

  source.configure(context);

}
 

import org.apache.flume.channel.MemoryChannel; //导入依赖的package包/类
private void init(String keepFields) {
  source = new SyslogTcpSource();
  channel = new MemoryChannel();

  Configurables.configure(channel, new Context());

  List<Channel> channels = new ArrayList<>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
  Context context = new Context();
  context.put("port", String.valueOf(TEST_SYSLOG_PORT));
  context.put("keepFields", keepFields);

  source.configure(context);

}
 

import org.apache.flume.channel.MemoryChannel; //导入依赖的package包/类
@Before
public void setUp() {
  source = new SpoolDirectorySource();
  channel = new MemoryChannel();

  Configurables.configure(channel, new Context());

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
  tmpDir = Files.createTempDir();
}
 

import org.apache.flume.channel.MemoryChannel; //导入依赖的package包/类
/**
 * We set up the the Netcat source and Flume Memory Channel on localhost
 *
 * @throws UnknownHostException
 */
@Before
public void setUp() throws UnknownHostException {
  localhost = InetAddress.getByName("127.0.0.1");
  source = new NetcatSource();
  channel = new MemoryChannel();

  Configurables.configure(channel, new Context());

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
}
 

import org.apache.flume.channel.MemoryChannel; //导入依赖的package包/类
@Before
public void setUp() {
  source = new TaildirSource();
  channel = new MemoryChannel();

  Configurables.configure(channel, new Context());

  List<Channel> channels = new ArrayList<Channel>();
  channels.add(channel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
  tmpDir = Files.createTempDir();
  posFilePath = tmpDir.getAbsolutePath() + "/taildir_position_test.json";
}
 

import org.apache.flume.channel.MemoryChannel; //导入依赖的package包/类
@BeforeClass
public static void setUpClass() throws Exception {
  port = findFreePort();
  Context context = new Context();
  context.put("port", String.valueOf(port));

  scribeSource = new ScribeSource();
  scribeSource.setName("Scribe Source");

  Configurables.configure(scribeSource, context);

  memoryChannel = new MemoryChannel();
  Configurables.configure(memoryChannel, context);

  List<Channel> channels = new ArrayList<Channel>(1);
  channels.add(memoryChannel);

  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  memoryChannel.start();

  scribeSource.setChannelProcessor(new ChannelProcessor(rcs));
  scribeSource.start();
}
 

import org.apache.flume.channel.MemoryChannel; //导入依赖的package包/类
@Test
public void testLifecycle() throws InterruptedException, LifecycleException {
  LOG.debug("Starting...");
  Context context = new Context();

  context.put("hdfs.path", testPath);
  /*
   * context.put("hdfs.rollInterval", String.class);
   * context.get("hdfs.rollSize", String.class); context.get("hdfs.rollCount",
   * String.class);
   */
  Configurables.configure(sink, context);

  sink.setChannel(new MemoryChannel());

  sink.start();
  sink.stop();
}
 

import org.apache.flume.channel.MemoryChannel; //导入依赖的package包/类
@Test (expected = EventDeliveryException.class)
public void testTimeOut() throws Exception {
  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  deleteTable = true;
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration(), true, false);
  Configurables.configure(sink, ctx);
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, ctx);
  sink.setChannel(channel);
  channel.start();
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for (int i = 0; i < 3; i++) {
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  Assert.assertFalse(sink.isConfNull());
  sink.process();
  Assert.fail();
}
 

import org.apache.flume.channel.MemoryChannel; //导入依赖的package包/类
@Test
public void testOneEvent() throws Exception {
  initContextForSimpleHbaseEventSerializer();
  HBaseSink sink = new HBaseSink(conf);
  Configurables.configure(sink, ctx);
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, new Context());
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  Event e = EventBuilder.withBody(
      Bytes.toBytes(valBase));
  channel.put(e);
  tx.commit();
  tx.close();

  sink.process();
  sink.stop();
  HTable table = new HTable(conf, tableName);
  byte[][] results = getResults(table, 1);
  byte[] out = results[0];
  Assert.assertArrayEquals(e.getBody(), out);
  out = results[1];
  Assert.assertArrayEquals(Longs.toByteArray(1), out);
}
 

import org.apache.flume.channel.MemoryChannel; //导入依赖的package包/类
@Test
public void testEmptyChannel() throws UnsupportedEncodingException, EventDeliveryException {
  Sink kafkaSink = new KafkaSink();
  Context context = prepareDefaultContext();
  Configurables.configure(kafkaSink, context);
  Channel memoryChannel = new MemoryChannel();
  Configurables.configure(memoryChannel, context);
  kafkaSink.setChannel(memoryChannel);
  kafkaSink.start();

  Sink.Status status = kafkaSink.process();
  if (status != Sink.Status.BACKOFF) {
    fail("Error Occurred");
  }
  assertNull(testUtil.getNextMessageFromConsumer(DEFAULT_TOPIC));
}
 

import org.apache.flume.channel.MemoryChannel; //导入依赖的package包/类
private Sink.Status prepareAndSend(Context context, String msg)
    throws EventDeliveryException {
  Sink kafkaSink = new KafkaSink();
  Configurables.configure(kafkaSink, context);
  Channel memoryChannel = new MemoryChannel();
  Configurables.configure(memoryChannel, context);
  kafkaSink.setChannel(memoryChannel);
  kafkaSink.start();

  Transaction tx = memoryChannel.getTransaction();
  tx.begin();
  Event event = EventBuilder.withBody(msg.getBytes());
  memoryChannel.put(event);
  tx.commit();
  tx.close();

  return kafkaSink.process();
}
 

import org.apache.flume.channel.MemoryChannel; //导入依赖的package包/类
@Before
public void setUp() {
  logger.info("Running setup");

  channel = new MemoryChannel();
  source = new NetcatSource();

  Context context = new Context();

  Configurables.configure(channel, context);
  List<Channel> channels = Lists.newArrayList(channel);
  ChannelSelector rcs = new ReplicatingChannelSelector();
  rcs.setChannels(channels);

  source.setChannelProcessor(new ChannelProcessor(rcs));
}