Java Sink类代码示例

import org.apache.flume.Sink; //导入依赖的package包/类
@Override
public Status process() throws EventDeliveryException {
  Status status = null;

  Iterator<Sink> sinkIterator = selector.createSinkIterator();
  while (sinkIterator.hasNext()) {
    Sink sink = sinkIterator.next();
    try {
      status = sink.process();
      break;
    } catch (Exception ex) {
      selector.informSinkFailed(sink);
      LOGGER.warn("Sink failed to consume event. "
          + "Attempting next sink if available.", ex);
    }
  }

  if (status == null) {
    throw new EventDeliveryException("All configured sinks have failed");
  }

  return status;
}
 

import org.apache.flume.Sink; //导入依赖的package包/类
@Test
public void testDuplicateCreate() {

  Sink avroSink1 = sinkFactory.create("avroSink1", "avro");
  Sink avroSink2 = sinkFactory.create("avroSink2", "avro");

  Assert.assertNotNull(avroSink1);
  Assert.assertNotNull(avroSink2);
  Assert.assertNotSame(avroSink1, avroSink2);
  Assert.assertTrue(avroSink1 instanceof AvroSink);
  Assert.assertTrue(avroSink2 instanceof AvroSink);

  Sink s1 = sinkFactory.create("avroSink1", "avro");
  Sink s2 = sinkFactory.create("avroSink2", "avro");

  Assert.assertNotSame(avroSink1, s1);
  Assert.assertNotSame(avroSink2, s2);
}
 

import org.apache.flume.Sink; //导入依赖的package包/类
@Test
public void testStaticTopic() {
  Context context = prepareDefaultContext();
  // add the static topic
  context.put(TOPIC_CONFIG, TestConstants.STATIC_TOPIC);
  String msg = "static-topic-test";

  try {
    Sink.Status status = prepareAndSend(context, msg);
    if (status == Sink.Status.BACKOFF) {
      fail("Error Occurred");
    }
  } catch (EventDeliveryException ex) {
    // ignore
  }

  String fetchedMsg = new String((byte[]) testUtil.getNextMessageFromConsumer(
      TestConstants.STATIC_TOPIC).message());
  assertEquals(msg, fetchedMsg);
}
 

import org.apache.flume.Sink; //导入依赖的package包/类
@Test
public void testEmptyChannel() throws UnsupportedEncodingException, EventDeliveryException {
  Sink kafkaSink = new KafkaSink();
  Context context = prepareDefaultContext();
  Configurables.configure(kafkaSink, context);
  Channel memoryChannel = new MemoryChannel();
  Configurables.configure(memoryChannel, context);
  kafkaSink.setChannel(memoryChannel);
  kafkaSink.start();

  Sink.Status status = kafkaSink.process();
  if (status != Sink.Status.BACKOFF) {
    fail("Error Occurred");
  }
  assertNull(testUtil.getNextMessageFromConsumer(DEFAULT_TOPIC));
}
 

import org.apache.flume.Sink; //导入依赖的package包/类
private Sink.Status prepareAndSend(Context context, String msg)
    throws EventDeliveryException {
  Sink kafkaSink = new KafkaSink();
  Configurables.configure(kafkaSink, context);
  Channel memoryChannel = new MemoryChannel();
  Configurables.configure(memoryChannel, context);
  kafkaSink.setChannel(memoryChannel);
  kafkaSink.start();

  Transaction tx = memoryChannel.getTransaction();
  tx.begin();
  Event event = EventBuilder.withBody(msg.getBytes());
  memoryChannel.put(event);
  tx.commit();
  tx.close();

  return kafkaSink.process();
}
 

import org.apache.flume.Sink; //导入依赖的package包/类
@Override
public void run() {
  run = true;
  while (run && count < until) {
    boolean error = true;
    try {
      if (Sink.Status.READY.equals(sink.process())) {
        count++;
        error = false;
      }
    } catch (Exception ex) {
      errors.add(ex);
    }
    if (error) {
      try {
        Thread.sleep(1000L);
      } catch (InterruptedException e) {
      }
    }
  }
}
 

import org.apache.flume.Sink; //导入依赖的package包/类
@Ignore @Test
public void processEmtpyChannel() throws EventDeliveryException {
  final CassandraSink sink = new CassandraSink();
  final Channel channel = mock(Channel.class);
  final Transaction tx = mock(Transaction.class);
  final CassandraTable table = mock(CassandraTable.class);
  final Context ctx = new Context();
  ctx.put("tables", "keyspace.table");
  sink.configure(ctx);
  sink.tables = Collections.singletonList(table);
  sink.setChannel(channel);
  when(channel.getTransaction()).thenReturn(tx);
  when(channel.take()).thenReturn(null);
  assertThat(sink.process()).isEqualTo(Sink.Status.READY);
  verifyZeroInteractions(table);
}
 

import org.apache.flume.Sink; //导入依赖的package包/类
@Ignore @Test
public void processOneEvent() throws EventDeliveryException {
  final CassandraSink sink = new CassandraSink();
  final Channel channel = mock(Channel.class);
  final Transaction tx = mock(Transaction.class);
  final CassandraTable table = mock(CassandraTable.class);
  final Context ctx = new Context();
  ctx.put("tables", "keyspace.table");
  sink.configure(ctx);
  sink.tables = Collections.singletonList(table);
  sink.setChannel(channel);
  when(channel.getTransaction()).thenReturn(tx);
  final Event event = EventBuilder.withBody(new byte[0], ImmutableMap.of("id", "1", "col", "text"));
  when(channel.take()).thenReturn(event).thenReturn(null);
  assertThat(sink.process()).isEqualTo(Sink.Status.READY);
  verify(table).save(ImmutableList.of(event));
  verifyNoMoreInteractions(table);
  verify(tx).begin();
  verify(tx).commit();
  verify(tx).close();
}
 

import org.apache.flume.Sink; //导入依赖的package包/类
@Test
public void processOneEventWithBatchSizeOne() throws EventDeliveryException {
  final CassandraSink sink = new CassandraSink();
  final Channel channel = mock(Channel.class);
  final Transaction tx = mock(Transaction.class);
  final CassandraTable table = mock(CassandraTable.class);
  final Context ctx = new Context();
  ctx.put("tables", "keyspace.table");
  ctx.put("batchSize", "1");
  sink.configure(ctx);
  sink.tables = Collections.singletonList(table);
  sink.setChannel(channel);
  when(channel.getTransaction()).thenReturn(tx);
  final Event event = EventBuilder.withBody(new byte[0], ImmutableMap.of("id", "1", "col", "text"));
  when(channel.take()).thenReturn(event).thenReturn(null);
  assertThat(sink.process()).isEqualTo(Sink.Status.READY);
  verify(table).save(ImmutableList.of(event));
  verifyNoMoreInteractions(table);
  verify(tx).begin();
  verify(tx).commit();
  verify(tx).close();
}
 

import org.apache.flume.Sink; //导入依赖的package包/类
private Sink moveActiveToDeadAndGetNext() {
  Integer key = liveSinks.lastKey();
  failedSinks.add(new FailedSink(key, activeSink, 1));
  liveSinks.remove(key);
  if (liveSinks.isEmpty()) return null;
  if (liveSinks.lastKey() != null) {
    return liveSinks.get(liveSinks.lastKey());
  } else {
    return null;
  }
}
 

import org.apache.flume.Sink; //导入依赖的package包/类
@Override
public void setSinks(List<Sink> sinks) {
  // needed to implement the start/stop functionality
  super.setSinks(sinks);

  this.sinks = new HashMap<String, Sink>();
  for (Sink sink : sinks) {
    this.sinks.put(sink.getName(), sink);
  }
}
 

import org.apache.flume.Sink; //导入依赖的package包/类
@Override
public void setSinks(List<Sink> sinks) {
  Preconditions.checkNotNull(sinks);
  Preconditions.checkArgument(sinks.size() == 1, "DefaultSinkPolicy can "
      + "only handle one sink, "
      + "try using a policy that supports multiple sinks");
  sink = sinks.get(0);
}
 

import org.apache.flume.Sink; //导入依赖的package包/类
@Override
public Sink create(String name, String type) throws FlumeException {
  Preconditions.checkNotNull(name, "name");
  Preconditions.checkNotNull(type, "type");
  logger.info("Creating instance of sink: {}, type: {}", name, type);
  Class<? extends Sink> sinkClass = getClass(type);
  try {
    Sink sink = sinkClass.newInstance();
    sink.setName(name);
    return sink;
  } catch (Exception ex) {
    throw new FlumeException("Unable to create sink: " + name
        + ", type: " + type + ", class: " + sinkClass.getName(), ex);
  }
}
 

import org.apache.flume.Sink; //导入依赖的package包/类
@Override
public void start() {
  for (Sink s : sinkList) {
    s.start();
  }

  state = LifecycleState.START;
}
 

import org.apache.flume.Sink; //导入依赖的package包/类
@Override
public void stop() {
  for (Sink s : sinkList) {
    s.stop();
  }
  state = LifecycleState.STOP;
}