Java EventDeserializer类代码示例

import org.apache.flume.serialization.EventDeserializer; //导入依赖的package包/类
private void validateMiniParse(EventDeserializer des) throws IOException {
  Event evt;

  des.mark();
  evt = des.readEvent();
  assertEquals(new String(evt.getBody()), mini);
  des.reset(); // reset!

  evt = des.readEvent();
  assertEquals("data should be repeated, " +
      "because we reset() the stream", new String(evt.getBody()), mini);

  evt = des.readEvent();
  assertNull("Event should be null because there are no lines " +
      "left to read", evt);

  des.mark();
  des.close();
}
 

import org.apache.flume.serialization.EventDeserializer; //导入依赖的package包/类
@Override
public EventDeserializer build(Context context, ResettableInputStream resettableInputStream) {
    Settings settings = createSettings(context);
    LDVFactory factory = new LDVFactory(settings);
    ResettableInputStreamWrapper wrapper = new ResettableInputStreamWrapper(resettableInputStream);
    LDVReader reader = factory.openReader(wrapper);

    Map<String, String> additionalHeaders = context.getSubProperties(Builder.HEADERS);
    boolean lowerCaseSchemaHeader=context.getBoolean("lowercase.header.schema", true);
    boolean lowerCaseTableHeader=context.getBoolean("lowercase.header.table", true);
    int bufferSize = context.getInteger("buffer.size", 256 * 1024);

    LDVEventBuilder eventBuilder = new LDVEventBuilder(additionalHeaders,
            bufferSize,
            lowerCaseSchemaHeader,
            lowerCaseTableHeader);

    return new LDVEventDeserializer(resettableInputStream, reader, eventBuilder);
}
 

import org.apache.flume.serialization.EventDeserializer; //导入依赖的package包/类
@Test
public void readEvents() throws Exception {
    Context context = new Context(params);
    LDVEventDeserializer.Builder builder = new LDVEventDeserializer.Builder();
    EventDeserializer eventDeserializer = builder.build(context, resettableFileInputStream);
    Assert.assertNotNull("eventDeserializer should not be null", eventDeserializer);
    List<Event> events = eventDeserializer.readEvents(expected.length * 2);
    Assert.assertEquals("expected array does not match events list",  expected.length, events.size());
    for(int i=0;i<expected.length;i++){
        Message message = expected[i];
        Event event = events.get(i);

        //Header size needs to account for the schema literal
        Assert.assertEquals(message.getMetadata().size() + 1, event.getHeaders().size());

        for(CharSequence key:message.getMetadata().keySet()){
            Assert.assertEquals(
                    String.format("Message at index %s: key %s does not match.", i, key),
                    message.getMetadata().get(key),
                    event.getHeaders().get(key)
            );
        }
    }
}
 

import org.apache.flume.serialization.EventDeserializer; //导入依赖的package包/类
private void validateReadAndMark(EventDeserializer des) throws IOException {
    Event evt;

    evt = des.readEvent();
    assertTrue(new String(evt.getBody()).contains("Giada De Laurentiis"));
    des.mark();

    evt = des.readEvent();
    assertTrue(new String(evt.getBody()).contains("J K. Rowling"));
    des.mark(); // reset!

    List<Event> readEvents = des.readEvents(2);
    assertEquals(2, readEvents.size());

    evt = des.readEvent();
    assertNull("Event should be null because there are no more books " + "left to read", evt);

    des.mark();
    des.mark();
    des.close();
}
 

import org.apache.flume.serialization.EventDeserializer; //导入依赖的package包/类
private void validateReadAndMarkWithHeader(EventDeserializer des) throws IOException {
    Event evt;

    evt = des.readEvent();
    System.out.println(evt.getHeaders().get("myHeader"));
    assertTrue(evt.getHeaders().get("myHeader").contains("Giada De Laurentiis"));
    des.mark();

    evt = des.readEvent();
    assertTrue(evt.getHeaders().get("myHeader").contains("J K. Rowling"));
    des.mark(); // reset!

    List<Event> readEvents = des.readEvents(2);
    assertEquals(2, readEvents.size());

    evt = des.readEvent();
    assertNull("Event should be null because there are no more books " + "left to read", evt);

    des.mark();
    des.mark();
    des.close();
}
 

import org.apache.flume.serialization.EventDeserializer; //导入依赖的package包/类
private void validateReset(EventDeserializer des) throws IOException {
    Event evt = des.readEvent();
    assertEquals("Everyday Italian", new String(evt.getBody()));
    des.mark();

    List<Event> events = des.readEvents(3);
    assertEquals(3, events.size());
    assertEquals("Harry Potter", new String(events.get(0).getBody()));
    assertEquals("XQuery Kick Start", new String(events.get(1).getBody()));
    assertEquals("Learning XML", new String(events.get(2).getBody()));

    des.reset(); // reset!

    events = des.readEvents(3);
    assertEquals(3, events.size());
    assertEquals("Harry Potter", new String(events.get(0).getBody()));
    assertEquals("XQuery Kick Start", new String(events.get(1).getBody()));
    assertEquals("Learning XML", new String(events.get(2).getBody()));

    evt = des.readEvent();
    Assert.assertNull("Event should be null because there are no more books " + "left to read", evt);

}
 

import org.apache.flume.serialization.EventDeserializer; //导入依赖的package包/类
private List<Event> readDeserializerEvents(int numEvents) throws IOException {
  EventDeserializer des = currentFile.get().getDeserializer();
  List<Event> events = des.readEvents(numEvents);
  if (events.isEmpty() && firstTimeRead) {
    events.add(EventBuilder.withBody(new byte[0]));
  }
  firstTimeRead = false;
  return events;
}
 

import org.apache.flume.serialization.EventDeserializer; //导入依赖的package包/类
@Test
public void testSimpleViaBuilder() throws IOException {
  ResettableInputStream in = new ResettableTestStringInputStream(mini);
  EventDeserializer.Builder builder = new BlobDeserializer.Builder();
  EventDeserializer des = builder.build(new Context(), in);
  validateMiniParse(des);
}
 

import org.apache.flume.serialization.EventDeserializer; //导入依赖的package包/类
@Test
public void testSimpleViaFactory() throws IOException {
  ResettableInputStream in = new ResettableTestStringInputStream(mini);
  EventDeserializer des;
  des = EventDeserializerFactory.getInstance(BlobDeserializer.Builder.class.getName(),
                                             new Context(), in);
  validateMiniParse(des);
}
 

import org.apache.flume.serialization.EventDeserializer; //导入依赖的package包/类
@Test
public void testBatch() throws IOException {
  ResettableInputStream in = new ResettableTestStringInputStream(mini);
  EventDeserializer des = new BlobDeserializer(new Context(), in);
  List<Event> events;

  events = des.readEvents(10); // try to read more than we should have
  assertEquals(1, events.size());
  assertEventBodyEquals(mini, events.get(0));

  des.mark();
  des.close();
}
 

import org.apache.flume.serialization.EventDeserializer; //导入依赖的package包/类
@Test
public void testMaxLineLength() throws IOException {
  String longLine = "abcdefghijklmnopqrstuvwxyz\n";
  Context ctx = new Context();
  ctx.put(BlobDeserializer.MAX_BLOB_LENGTH_KEY, "10");

  ResettableInputStream in = new ResettableTestStringInputStream(longLine);
  EventDeserializer des = new BlobDeserializer(ctx, in);

  assertEventBodyEquals("abcdefghij", des.readEvent());
  assertEventBodyEquals("klmnopqrst", des.readEvent());
  assertEventBodyEquals("uvwxyz\n", des.readEvent());
  assertNull(des.readEvent());
}
 

import org.apache.flume.serialization.EventDeserializer; //导入依赖的package包/类
@Override
public EventDeserializer build(Context context, ResettableInputStream in) {
    if (!(in instanceof Seekable)) {
        throw new IllegalArgumentException(
                "Cannot use this deserializer without a Seekable input stream");
    }
    try {
        return new XmlXpathDeserializer(context, in);
    } catch (IOException ex) {
        throw new RuntimeException(ex);
    }
}
 

import org.apache.flume.serialization.EventDeserializer; //导入依赖的package包/类
@Test
public void testReadsAndMark() throws IOException {
    Context context = new Context();
    context.put("expression", "/bookstore/book");
    EventDeserializer des = new XmlXpathDeserializer.Builder().build(context, getTestInputStream());
    validateReadAndMark(des);
}
 

import org.apache.flume.serialization.EventDeserializer; //导入依赖的package包/类
@Test
public void testReset() throws IOException {
    Context context = new Context();
    context.put("expression", "/bookstore/book/title/text()");
    EventDeserializer des = new XmlXpathDeserializer.Builder().build(context, getTestInputStream());
    validateReset(des);
}
 

import org.apache.flume.serialization.EventDeserializer; //导入依赖的package包/类
@Test
public void testHeader() throws IOException {
  Context context = new Context();
  context.put("expression", "/bookstore/book");
  context.put("outputHeader", "myHeader");
  context.put("outputBody", "false");
  EventDeserializer des = new XmlXpathDeserializer.Builder().build(context, getTestInputStream());
  validateReadAndMarkWithHeader(des);
}