Java EventSerializer类代码示例

import org.apache.flume.serialization.EventSerializer; //导入依赖的package包/类
@Test
public void testWithNewline() throws FileNotFoundException, IOException {

    Map<String, String> headers = new HashMap<String, String>();
    headers.put("message", "message1");
    OutputStream out = new FileOutputStream(testFile);
    CustomLastfmHeaderAndBodyTextEventSerializer.Builder builder = CustomLastfmHeaderAndBodyTextEventSerializer.builder();
    EventSerializer serializer = builder.build(new Context(), out);
    serializer.afterCreate();
    serializer.write(EventBuilder.withBody("messageBody", Charsets.UTF_8, headers));
    serializer.flush();
    serializer.beforeClose();
    out.flush();
    out.close();

    BufferedReader reader = new BufferedReader(new FileReader(testFile));
    Assert.assertEquals("message1", reader.readLine());
    Assert.assertNull(reader.readLine());
    reader.close();

    FileUtils.forceDelete(testFile);
}
 

import org.apache.flume.serialization.EventSerializer; //导入依赖的package包/类
@Test
public void testNoNewline() throws FileNotFoundException, IOException {

    Map<String, String> headers = new HashMap<String, String>();
    headers.put("header1", "value1");

    OutputStream out = new FileOutputStream(testFile);
    Context context = new Context();
    context.put("appendNewline", "false");
    CustomLastfmHeaderAndBodyTextEventSerializer.Builder builder = CustomLastfmHeaderAndBodyTextEventSerializer.builder();
    EventSerializer serializer = builder.build(new Context(), out);
    serializer.afterCreate();
    serializer.write(EventBuilder.withBody("event 1", Charsets.UTF_8, headers));
    serializer.write(EventBuilder.withBody("event 2", Charsets.UTF_8, headers));
    serializer.write(EventBuilder.withBody("event 3", Charsets.UTF_8, headers));
    serializer.flush();
    serializer.beforeClose();
    out.flush();
    out.close();

    BufferedReader reader = new BufferedReader(new FileReader(testFile));
    Assert.assertNull(reader.readLine());
    reader.close();

    FileUtils.forceDelete(testFile);
}
 

import org.apache.flume.serialization.EventSerializer; //导入依赖的package包/类
@Override
public void configure(Context context) {
  super.configure(context);

  serializerType = context.getString("serializer", "TEXT");
  useRawLocalFileSystem = context.getBoolean("hdfs.useRawLocalFileSystem",
      false);
  serializerContext =
      new Context(context.getSubProperties(EventSerializer.CTX_PREFIX));
  logger.info("Serializer = " + serializerType + ", UseRawLocalFileSystem = "
      + useRawLocalFileSystem);
}
 

import org.apache.flume.serialization.EventSerializer; //导入依赖的package包/类
@Override
public void configure(Context context) {
  super.configure(context);

  serializerType = context.getString("serializer", "TEXT");
  useRawLocalFileSystem = context.getBoolean("hdfs.useRawLocalFileSystem",
      false);
  serializerContext = new Context(
      context.getSubProperties(EventSerializer.CTX_PREFIX));
  logger.info("Serializer = " + serializerType + ", UseRawLocalFileSystem = "
      + useRawLocalFileSystem);
}
 

import org.apache.flume.serialization.EventSerializer; //导入依赖的package包/类
@Override
public EventSerializer build(Context context, OutputStream out) {
    path = context.getString(PATH, PATH_DEFAULT);
    customerHeader = context.getString(CUSTOMER_HEADER, CUSTOMER_HEADER_DEFAULT);
    hostHeader = context.getString(HOST_HEADER, HOST_HEADER_DEFAULT);
    SyslogAvroEventSerializer writer = null;
    try {
        writer = new SyslogAvroEventSerializer(out, path, customerHeader, hostHeader);
        writer.configure(context);
    } catch (IOException e) {
        log.error("Unable to parse schema file. Exception follows.", e);
    }
    return writer;
}
 

import org.apache.flume.serialization.EventSerializer; //导入依赖的package包/类
@Override
public EventSerializer build(Context context, OutputStream out) {
    ApacheLogAvroEventSerializer writer = null;
    try {
        writer = new ApacheLogAvroEventSerializer(out);
        writer.configure(context);
    } catch (IOException e) {
        log.error("Unable to parse schema file. Exception follows.", e);
    }
    return writer;
}
 

import org.apache.flume.serialization.EventSerializer; //导入依赖的package包/类
@Override
public EventSerializer build(Context context, OutputStream out) {
    JavaLogAvroEventSerializer writer = null;
    try {
        writer = new JavaLogAvroEventSerializer(out);
        writer.configure(context);
    } catch (IOException e) {
        log.error("Unable to parse schema file. Exception follows.", e);
    }
    return writer;
}
 

import org.apache.flume.serialization.EventSerializer; //导入依赖的package包/类
@Override
public EventSerializer build(Context context, OutputStream out) {
  FlumeEventAvroEventSerializer writer = new FlumeEventAvroEventSerializer(out);
  writer.configure(context);
  return writer;
}
 

import org.apache.flume.serialization.EventSerializer; //导入依赖的package包/类
@Override
public EventSerializer build(Context context, OutputStream out) {
  AvroEventSerializer writer = new AvroEventSerializer(out);
  writer.configure(context);
  return writer;
}
 

import org.apache.flume.serialization.EventSerializer; //导入依赖的package包/类
public void createAvroFile(File file, String codec, boolean useSchemaUrl,
                           boolean useStaticSchemaUrl) throws IOException {
  // serialize a few events using the reflection-based avro serializer
  OutputStream out = new FileOutputStream(file);

  Context ctx = new Context();
  if (codec != null) {
    ctx.put("compressionCodec", codec);
  }

  Schema schema = Schema.createRecord("myrecord", null, null, false);
  schema.setFields(Arrays.asList(new Schema.Field[]{
      new Schema.Field("message", Schema.create(Schema.Type.STRING), null, null)
  }));
  GenericRecordBuilder recordBuilder = new GenericRecordBuilder(schema);
  File schemaFile = null;
  if (useSchemaUrl || useStaticSchemaUrl) {
    schemaFile = File.createTempFile(getClass().getSimpleName(), ".avsc");
    Files.write(schema.toString(), schemaFile, Charsets.UTF_8);
  }

  if (useStaticSchemaUrl) {
    ctx.put(AvroEventSerializerConfigurationConstants.STATIC_SCHEMA_URL,
            schemaFile.toURI().toURL().toExternalForm());
  }

  EventSerializer.Builder builder = new AvroEventSerializer.Builder();
  EventSerializer serializer = builder.build(ctx, out);

  serializer.afterCreate();
  for (int i = 0; i < 3; i++) {
    GenericRecord record = recordBuilder.set("message", "Hello " + i).build();
    Event event = EventBuilder.withBody(serializeAvro(record, schema));
    if (schemaFile == null && !useSchemaUrl) {
      event.getHeaders().put(AvroEventSerializer.AVRO_SCHEMA_LITERAL_HEADER,
          schema.toString());
    } else if (useSchemaUrl) {
      event.getHeaders().put(AvroEventSerializer.AVRO_SCHEMA_URL_HEADER,
          schemaFile.toURI().toURL().toExternalForm());
    }
    serializer.write(event);
  }
  serializer.flush();
  serializer.beforeClose();
  out.flush();
  out.close();
  if (schemaFile != null ) {
    schemaFile.delete();
  }

}
 

import org.apache.flume.serialization.EventSerializer; //导入依赖的package包/类
@Override
public EventSerializer build(Context context, OutputStream out) {
    MessageAvroEventSerializer writer = new MessageAvroEventSerializer(out);
    writer.configure(context);
    return writer;
}
 

import org.apache.flume.serialization.EventSerializer; //导入依赖的package包/类
@Override
public EventSerializer build(Context context, OutputStream out) {
    return new CustomLastfmHeaderAndBodyTextEventSerializer(out, context);
}
 

import org.apache.flume.serialization.EventSerializer; //导入依赖的package包/类
@Override
public EventSerializer build(Context context, OutputStream out) {
  AvroKaaEventSerializer writer = new AvroKaaEventSerializer(out);
  writer.configure(context);
  return writer;
}
 

import org.apache.flume.serialization.EventSerializer; //导入依赖的package包/类
@Override
public EventSerializer build(Context context, OutputStream out) {
  CSVAvroSerializer ser = new CSVAvroSerializer(out);
  ser.configure(context);
  return ser;
}
 

import org.apache.flume.serialization.EventSerializer; //导入依赖的package包/类
@Override
public EventSerializer build(Context context, OutputStream out) {
    FlumeEventStringAvroEventSerializer writer = new FlumeEventStringAvroEventSerializer(out);
    writer.configure(context);
    return writer;
}