Java Status.BACKOFF属性代码示例

@Test
public void testDefaultConfiguration() throws Exception {
  // If no selector is specified, the round-robin selector should be used
  Channel ch = new MockChannel();
  int n = 100;
  int numEvents = 3 * n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);

  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor(sinks, new Context());

  Status s = Status.READY;
  while (s != Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertTrue(s1.getEvents().size() == n);
  Assert.assertTrue(s2.getEvents().size() == n);
  Assert.assertTrue(s3.getEvents().size() == n);

}
 

@Test
public void testRandomPersistentFailure() throws Exception {
  Channel ch = new MockChannel();
  int n = 100;
  int numEvents = 3 * n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);

  // s2 always fails
  s2.setFail(true);

  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor("random",sinks, false);

  Status s = Status.READY;
  while (s != Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertTrue(s2.getEvents().size() == 0);
  Assert.assertTrue(s1.getEvents().size() + s3.getEvents().size() == 3 * n);
}
 

@Test
public void testRoundRobinPersistentFailure() throws Exception {
  Channel ch = new MockChannel();
  int n = 100;
  int numEvents = 3 * n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);

  // s2 always fails
  s2.setFail(true);

  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor("round_robin",sinks, false);

  Status s = Status.READY;
  while (s != Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertTrue(s1.getEvents().size() == n);
  Assert.assertTrue(s2.getEvents().size() == 0);
  Assert.assertTrue(s3.getEvents().size() == 2 * n);
}
 

@Test
public void testRoundRobinNoFailure() throws Exception {

  Channel ch = new MockChannel();
  int n = 100;
  int numEvents = 3 * n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);

  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor("round_robin",sinks, false);

  Status s = Status.READY;
  while (s != Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertTrue(s1.getEvents().size() == n);
  Assert.assertTrue(s2.getEvents().size() == n);
  Assert.assertTrue(s3.getEvents().size() == n);
}
 

@Override
public Status process() throws EventDeliveryException {
  if (fail) {
    throw new EventDeliveryException("failed");
  }
  Event e = this.getChannel().take();
  if (e == null) {
    return Status.BACKOFF;
  }

  events.add(e);
  return Status.READY;
}
 

@Test
public void shouldIndexFiveEventsOverThreeBatches() throws Exception {
  parameters.put(BATCH_SIZE, "2");
  Configurables.configure(fixture, new Context(parameters));
  Channel channel = bindAndStartChannel(fixture);

  int numberOfEvents = 5;
  Event[] events = new Event[numberOfEvents];

  Transaction tx = channel.getTransaction();
  tx.begin();
  for (int i = 0; i < numberOfEvents; i++) {
    String body = "event #" + i + " of " + numberOfEvents;
    Event event = EventBuilder.withBody(body.getBytes());
    events[i] = event;
    channel.put(event);
  }
  tx.commit();
  tx.close();

  int count = 0;
  Status status = Status.READY;
  while (status != Status.BACKOFF) {
    count++;
    status = fixture.process();
  }
  fixture.stop();

  assertEquals(3, count);

  client.admin().indices()
      .refresh(Requests.refreshRequest(timestampedIndexName)).actionGet();
  assertMatchAllQuery(numberOfEvents, events);
  assertBodyQuery(5, events);
}
 

@Override
public Status process() throws EventDeliveryException {
    if (invalidConfiguration) {
        return Status.BACKOFF;
    } else if (rollbackedAccumulations.isEmpty()) {
        return processNewBatches();
    } else {
        processRollbackedBatches();
        return processNewBatches();
    } // if else
}
 

@Ignore @Test
public void shouldIndexFiveEventsOverThreeBatches() throws Exception {
  parameters.put(BATCH_SIZE, "2");
  Configurables.configure(fixture, new Context(parameters));
  Channel channel = bindAndStartChannel(fixture);

  int numberOfEvents = 5;
  Event[] events = new Event[numberOfEvents];

  Transaction tx = channel.getTransaction();
  tx.begin();
  for (int i = 0; i < numberOfEvents; i++) {
    String body = "event #" + i + " of " + numberOfEvents;
    Event event = EventBuilder.withBody(body.getBytes());
    events[i] = event;
    channel.put(event);
  }
  tx.commit();
  tx.close();

  int count = 0;
  Status status = Status.READY;
  while (status != Status.BACKOFF) {
    count++;
    status = fixture.process();
  }
  fixture.stop();

  assertEquals(3, count);

  client.admin().indices()
      .refresh(Requests.refreshRequest(timestampedIndexName)).actionGet();
  assertMatchAllQuery(numberOfEvents, events);
  assertBodyQuery(5, events);
}
 

@Test
public void testRandomNoFailure() throws Exception {

  Channel ch = new MockChannel();
  int n = 10000;
  int numEvents = n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);

  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  MockSink s4 = new MockSink(4);
  s4.setChannel(ch);

  MockSink s5 = new MockSink(5);
  s5.setChannel(ch);

  MockSink s6 = new MockSink(6);
  s6.setChannel(ch);

  MockSink s7 = new MockSink(7);
  s7.setChannel(ch);

  MockSink s8 = new MockSink(8);
  s8.setChannel(ch);

  MockSink s9 = new MockSink(9);
  s9.setChannel(ch);

  MockSink s0 = new MockSink(0);
  s0.setChannel(ch);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);
  sinks.add(s4);
  sinks.add(s5);
  sinks.add(s6);
  sinks.add(s7);
  sinks.add(s8);
  sinks.add(s9);
  sinks.add(s0);

  LoadBalancingSinkProcessor lbsp = getProcessor("random",sinks, false);

  Status s = Status.READY;
  while (s != Status.BACKOFF) {
    s = lbsp.process();
  }

  Set<Integer> sizeSet = new HashSet<Integer>();
  int sum = 0;
  for (Sink ms : sinks) {
    int count = ((MockSink) ms).getEvents().size();
    sum += count;
    sizeSet.add(count);
  }

  // Assert that all the events were accounted for
  Assert.assertEquals(n, sum);

  // Assert that at least two sinks came with different event sizes.
  // This makes sense if the total number of events is evenly divisible by
  // the total number of sinks. In which case the round-robin policy will
  // end up causing all sinks to get the same number of events where as
  // the random policy will have very low probability of doing that.
  Assert.assertTrue("Miraculous distribution", sizeSet.size() > 1);
}
 

@Test
public void testRoundRobinBackoffInitialFailure() throws EventDeliveryException {
  Channel ch = new MockChannel();
  int n = 100;
  int numEvents = 3 * n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);

  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor("round_robin",sinks, true);

  Status s = Status.READY;
  for (int i = 0; i < 3 && s != Status.BACKOFF; i++) {
    s = lbsp.process();
  }
  s2.setFail(true);
  for (int i = 0; i < 3 && s != Status.BACKOFF; i++) {
    s = lbsp.process();
  }
  s2.setFail(false);
  while (s != Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertEquals((3 * n) / 2, s1.getEvents().size());
  Assert.assertEquals(1, s2.getEvents().size());
  Assert.assertEquals((3 * n) / 2 - 1, s3.getEvents().size());
}
 

@Test
public void testRoundRobinBackoffIncreasingBackoffs()
    throws EventDeliveryException, InterruptedException {
  Channel ch = new MockChannel();
  int n = 100;
  int numEvents = 3 * n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);
  s2.setFail(true);

  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor("round_robin",sinks, true);

  Status s = Status.READY;
  for (int i = 0; i < 3 && s != Status.BACKOFF; i++) {
    s = lbsp.process();
  }
  Assert.assertEquals(0, s2.getEvents().size());
  Thread.sleep(2100);
  // this should let the sink come out of backoff and get backed off  for a longer time
  for (int i = 0; i < 3 && s != Status.BACKOFF; i++) {
    s = lbsp.process();
  }
  Assert.assertEquals(0, s2.getEvents().size());
  s2.setFail(false);
  Thread.sleep(2100);
  // this time it shouldn't come out of backoff yet as the timeout isn't over
  for (int i = 0; i < 3 && s != Status.BACKOFF; i++) {
    s = lbsp.process();
  }
  Assert.assertEquals(0, s2.getEvents().size());
  // after this s2 should be receiving events agains
  Thread.sleep(2100);
  while (s != Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertEquals( n + 2, s1.getEvents().size());
  Assert.assertEquals( n - 3, s2.getEvents().size());
  Assert.assertEquals( n + 1, s3.getEvents().size());
}
 

@Test
public void testRoundRobinBackoffFailureRecovery()
    throws EventDeliveryException, InterruptedException {
  Channel ch = new MockChannel();
  int n = 100;
  int numEvents = 3 * n;
  for (int i = 0; i < numEvents; i++) {
    ch.put(new MockEvent("test" + i));
  }

  MockSink s1 = new MockSink(1);
  s1.setChannel(ch);

  MockSink s2 = new MockSink(2);
  s2.setChannel(ch);
  s2.setFail(true);

  MockSink s3 = new MockSink(3);
  s3.setChannel(ch);

  List<Sink> sinks = new ArrayList<Sink>();
  sinks.add(s1);
  sinks.add(s2);
  sinks.add(s3);

  LoadBalancingSinkProcessor lbsp = getProcessor("round_robin",sinks, true);

  Status s = Status.READY;
  for (int i = 0; i < 3 && s != Status.BACKOFF; i++) {
    s = lbsp.process();
  }
  s2.setFail(false);
  Thread.sleep(2001);
  while (s != Status.BACKOFF) {
    s = lbsp.process();
  }

  Assert.assertEquals(n + 1, s1.getEvents().size());
  Assert.assertEquals(n - 1,  s2.getEvents().size());
  Assert.assertEquals(n, s3.getEvents().size());
}
 

@Test
public void testMultipleBatches() throws Exception {
  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  deleteTable = true;
  ctx.put("batchSize", "2");
  AsyncHBaseSink sink = new AsyncHBaseSink(testUtility.getConfiguration());
  Configurables.configure(sink, ctx);
  //Reset the context to a higher batchSize
  ctx.put("batchSize", "100");
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, ctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for (int i = 0; i < 3; i++) {
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  int count = 0;
  Status status = Status.READY;
  while (status != Status.BACKOFF) {
    count++;
    status = sink.process();
  }
  Assert.assertFalse(sink.isConfNull());
  sink.stop();
  Assert.assertEquals(2, count);
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 3);
  byte[] out;
  int found = 0;
  for (int i = 0; i < 3; i++) {
    for (int j = 0; j < 3; j++) {
      if (Arrays.equals(results[j], Bytes.toBytes(valBase + "-" + i))) {
        found++;
        break;
      }
    }
  }
  Assert.assertEquals(3, found);
  out = results[3];
  Assert.assertArrayEquals(Longs.toByteArray(3), out);
}
 

@Test
public void testWithoutConfigurationObject() throws Exception {
  testUtility.createTable(tableName.getBytes(), columnFamily.getBytes());
  deleteTable = true;
  ctx.put("batchSize", "2");
  ctx.put(HBaseSinkConfigurationConstants.ZK_QUORUM,
          ZKConfig.getZKQuorumServersString(testUtility.getConfiguration()));
  ctx.put(HBaseSinkConfigurationConstants.ZK_ZNODE_PARENT,
          testUtility.getConfiguration().get(HConstants.ZOOKEEPER_ZNODE_PARENT));
  AsyncHBaseSink sink = new AsyncHBaseSink();
  Configurables.configure(sink, ctx);
  // Reset context to values usable by other tests.
  ctx.put(HBaseSinkConfigurationConstants.ZK_QUORUM, null);
  ctx.put(HBaseSinkConfigurationConstants.ZK_ZNODE_PARENT, null);
  ctx.put("batchSize", "100");
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, ctx);
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for (int i = 0; i < 3; i++) {
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  int count = 0;
  Status status = Status.READY;
  while (status != Status.BACKOFF) {
    count++;
    status = sink.process();
  }
  /*
   * Make sure that the configuration was picked up from the context itself
   * and not from a configuration object which was created by the sink.
   */
  Assert.assertTrue(sink.isConfNull());
  sink.stop();
  Assert.assertEquals(2, count);
  HTable table = new HTable(testUtility.getConfiguration(), tableName);
  byte[][] results = getResults(table, 3);
  byte[] out;
  int found = 0;
  for (int i = 0; i < 3; i++) {
    for (int j = 0; j < 3; j++) {
      if (Arrays.equals(results[j], Bytes.toBytes(valBase + "-" + i))) {
        found++;
        break;
      }
    }
  }
  Assert.assertEquals(3, found);
  out = results[3];
  Assert.assertArrayEquals(Longs.toByteArray(3), out);
}
 

@Test
public void testMultipleBatches() throws Exception {
  initContextForSimpleHbaseEventSerializer();
  ctx.put("batchSize", "2");
  HBaseSink sink = new HBaseSink(conf);
  Configurables.configure(sink, ctx);
  //Reset the context to a higher batchSize
  ctx.put("batchSize", "100");
  Channel channel = new MemoryChannel();
  Configurables.configure(channel, new Context());
  sink.setChannel(channel);
  sink.start();
  Transaction tx = channel.getTransaction();
  tx.begin();
  for (int i = 0; i < 3; i++) {
    Event e = EventBuilder.withBody(Bytes.toBytes(valBase + "-" + i));
    channel.put(e);
  }
  tx.commit();
  tx.close();
  int count = 0;
  while (sink.process() != Status.BACKOFF) {
    count++;
  }
  sink.stop();
  Assert.assertEquals(2, count);
  HTable table = new HTable(conf, tableName);
  byte[][] results = getResults(table, 3);
  byte[] out;
  int found = 0;
  for (int i = 0; i < 3; i++) {
    for (int j = 0; j < 3; j++) {
      if (Arrays.equals(results[j], Bytes.toBytes(valBase + "-" + i))) {
        found++;
        break;
      }
    }
  }
  Assert.assertEquals(3, found);
  out = results[3];
  Assert.assertArrayEquals(Longs.toByteArray(3), out);
}