Java DatumWriter类代码示例

import org.apache.avro.io.DatumWriter; //导入依赖的package包/类
static void convertAvroToJson(InputStream inputStream, OutputStream outputStream, Schema schema)
        throws IOException {
    DatumReader<Object> reader = new GenericDatumReader<>(schema);
    DatumWriter<Object> writer = new GenericDatumWriter<>(schema);

    BinaryDecoder binaryDecoder = DecoderFactory.get().binaryDecoder(inputStream, null);

    JsonEncoder jsonEncoder = EncoderFactory.get().jsonEncoder(schema, outputStream, true);
    Object datum = null;
    while (!binaryDecoder.isEnd()) {
        datum = reader.read(datum, binaryDecoder);
        writer.write(datum, jsonEncoder);
        jsonEncoder.flush();
    }
    outputStream.flush();
}
 

import org.apache.avro.io.DatumWriter; //导入依赖的package包/类
/**
 * Encode the message data provided.
 * 
 * @param <O> The type of the data to encode.
 * @param data The message data.
 * @throws EncodeMessageContentException Exception thrown if encoding the message content fails.
 */
public <O extends GenericContainer> void encode(O data) throws EncodeMessageContentException {

	try {

		ByteArrayOutputStream baos = new ByteArrayOutputStream();

		this.schema = data.getSchema();
		DatumWriter<O> outputDatumWriter = new SpecificDatumWriter<O>(this.schema);
		BinaryEncoder encoder = EncoderFactory.get().binaryEncoder(baos, null);
		outputDatumWriter.write(data, encoder);
		encoder.flush();

		this.data = baos.toByteArray();

	} catch (Exception ex) {
		throw new EncodeMessageContentException(ex);
	}

}
 

import org.apache.avro.io.DatumWriter; //导入依赖的package包/类
public void jsonReadWriteExample() throws IOException {
	Employee employee = Employee.newBuilder().setFirstName("Gaurav")
			.setLastName("Mazra").setSex(SEX.MALE).build();

	DatumWriter<Employee> employeeWriter = new SpecificDatumWriter<>(Employee.class);
	byte[] data;
	try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
		Encoder jsonEncoder = EncoderFactory.get().jsonEncoder(Employee.getClassSchema(), baos);
		employeeWriter.write(employee, jsonEncoder);
		jsonEncoder.flush();
		data = baos.toByteArray();
	}
	
	// serialized data
	System.out.println(new String(data));
	
	DatumReader<Employee> employeeReader = new SpecificDatumReader<>(Employee.class);
	Decoder decoder = DecoderFactory.get().jsonDecoder(Employee.getClassSchema(), new String(data));
	employee = employeeReader.read(null, decoder);
	//data after deserialization
	System.out.println(employee);
}
 

import org.apache.avro.io.DatumWriter; //导入依赖的package包/类
public void binaryReadWriteExample() throws IOException {
	Employee employee = Employee.newBuilder().setFirstName("Gaurav")
			.setLastName("Mazra").setSex(SEX.MALE).build();

	DatumWriter<Employee> employeeWriter = new SpecificDatumWriter<>(Employee.class);
	byte[] data;
	try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
		Encoder binaryEncoder = EncoderFactory.get().binaryEncoder(baos, null);
		employeeWriter.write(employee, binaryEncoder);
		binaryEncoder.flush();
		data = baos.toByteArray();
	}
	
	// serialized data
	System.out.println(data);
	
	DatumReader<Employee> employeeReader = new SpecificDatumReader<>(Employee.class);
	Decoder binaryDecoder = DecoderFactory.get().binaryDecoder(data, null);
	employee = employeeReader.read(null, binaryDecoder);
	//data after deserialization
	System.out.println(employee);
}
 

import org.apache.avro.io.DatumWriter; //导入依赖的package包/类
/**
 * Process singlex.
 *
 * @throws Exception the exception
 */
public void processSinglex() throws Exception {
	int base = (int) System.currentTimeMillis();
	User user = User.newBuilder().setName("name" + base).setFavoriteColor("color" + base).setFavoriteNumber(base)
			.build();
	DatumWriter<GenericRecord> datumWriterUser = new GenericDatumWriter<GenericRecord>(User.getClassSchema());
	ByteArrayOutputStream baos = new ByteArrayOutputStream();
	byte[] byteData = null;
	try {
		BinaryEncoder binaryEncoder = EncoderFactory.get().binaryEncoder(baos, null);
		datumWriterUser.write(user, binaryEncoder);
		binaryEncoder.flush();
		byteData = baos.toByteArray();
	} finally {
		baos.close();
	}
	System.out.println(byteData.length);
	
	DatumReader<GenericRecord> datumReaderUser = new GenericDatumReader<GenericRecord>( User.getClassSchema());
	GenericRecord genericRecord = datumReaderUser.read(null, DecoderFactory.get().binaryDecoder(byteData, null) );
	System.out.println(genericRecord);
	System.out.println( genericRecord.get("name"));
}
 

import org.apache.avro.io.DatumWriter; //导入依赖的package包/类
@Override
public String getJsonStringFromRecord(RecordField field)
        throws AvroUiSandboxServiceException {
    try {
        GenericRecord record = FormAvroConverter.createGenericRecordFromRecordField(field);
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        JsonGenerator jsonGenerator
            = new JsonFactory().createJsonGenerator(baos, JsonEncoding.UTF8);
        jsonGenerator.useDefaultPrettyPrinter();
        JsonEncoder jsonEncoder = EncoderFactory.get().jsonEncoder(record.getSchema(), jsonGenerator);
        DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(record.getSchema());
        datumWriter.write(record, jsonEncoder);
        jsonEncoder.flush();
        baos.flush();            
        return new String(baos.toByteArray(), UTF8);
    } catch (Exception e) {
        throw Utils.handleException(e);
    }
}
 

import org.apache.avro.io.DatumWriter; //导入依赖的package包/类
@Override
public byte[] serialize(String topic, T payload) {
	try {
		byte[] result = null;

		if (payload != null) {
			LOGGER.debug("data='{}'", payload);

			ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
			BinaryEncoder binaryEncoder = EncoderFactory.get().binaryEncoder(byteArrayOutputStream, null);

			DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<>(payload.getSchema());
			datumWriter.write(payload, binaryEncoder);

			binaryEncoder.flush();
			byteArrayOutputStream.close();

			result = byteArrayOutputStream.toByteArray();
			
			LOGGER.debug("serialized data='{}'", DatatypeConverter.printHexBinary(result));
		}
		return result;
	} catch (IOException ex) {
		throw new SerializationException("Can't serialize payload='" + payload + "' for topic='" + topic + "'", ex);
	}
}
 

import org.apache.avro.io.DatumWriter; //导入依赖的package包/类
private static Path createDataFile() throws IOException {
    File avroFile = File.createTempFile("test-", "." + FILE_EXTENSION);
    DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema);
    try (DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<>(writer)) {
        dataFileWriter.setFlushOnEveryBlock(true);
        dataFileWriter.setSyncInterval(32);
        dataFileWriter.create(schema, avroFile);

        IntStream.range(0, NUM_RECORDS).forEach(index -> {
            GenericRecord datum = new GenericData.Record(schema);
            datum.put(FIELD_INDEX, index);
            datum.put(FIELD_NAME, String.format("%d_name_%s", index, UUID.randomUUID()));
            datum.put(FIELD_SURNAME, String.format("%d_surname_%s", index, UUID.randomUUID()));
            try {
                OFFSETS_BY_INDEX.put(index, dataFileWriter.sync() - 16L);
                dataFileWriter.append(datum);
            } catch (IOException ioe) {
                throw new RuntimeException(ioe);
            }
        });
    }
    Path path = new Path(new Path(fsUri), avroFile.getName());
    fs.moveFromLocalFile(new Path(avroFile.getAbsolutePath()), path);
    return path;
}
 

import org.apache.avro.io.DatumWriter; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
	Employee employee = Employee.newBuilder().setFirstName("Gaurav")
			.setLastName("Mazra").setSex(SEX.MALE).build();
	
	byte[] payload;
	DatumWriter<Employee> datumWriter = new SpecificDatumWriter<>(Employee.class);
	try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
		Encoder out = EncoderFactory.get().binaryEncoder(baos, null);
		datumWriter.write(employee, out );
		out.flush();
		payload = baos.toByteArray();
	}
	catch (Exception e) {
		System.err.println(e);
		throw e;
	}
	System.out.println(new String(payload));
	System.out.println(payload.length);
	
}
 

import org.apache.avro.io.DatumWriter; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
	Node node = new Node();
	node.setValue("Gaurav");
	node.setNext(node);
	
	byte[] payload;
	DatumWriter<Node> datumWriter = new ReflectDatumWriter<>(Node.class);
	try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
		Encoder out = EncoderFactory.get().binaryEncoder(baos, null);
		datumWriter.write(node, out );
		out.flush();
		payload = baos.toByteArray();
	}
	catch (Exception e) {
		System.err.println(e);
		throw e;
	}
	System.out.println(new String(payload));
	System.out.println(payload.length);
}
 

import org.apache.avro.io.DatumWriter; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
	Node node = new Node();
	node.setValue("Gaurav");
	node.setNext(node);
	
	byte[] payload;
	DatumWriter<Node> datumWriter = new SpecificDatumWriter<>(Node.class);
	try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
		Encoder out = EncoderFactory.get().binaryEncoder(baos, null);
		datumWriter.write(node, out );
		out.flush();
		payload = baos.toByteArray();
	}
	catch (Exception e) {
		System.err.println(e);
		throw e;
	}
	System.out.println(new String(payload));
	System.out.println(payload.length);
}
 

import org.apache.avro.io.DatumWriter; //导入依赖的package包/类
/**
 * tests Avro Serializer
 */
@Test
public void testSerializer() throws Exception {
    Context context = new Context();
    String schemaFile = getClass().getResource("/schema.avsc").getFile();
    context.put(ES_AVRO_SCHEMA_FILE, schemaFile);
    avroSerializer.configure(context);
    Schema schema = new Schema.Parser().parse(new File(schemaFile));
    GenericRecord user = generateGenericRecord(schema);
    DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(schema);
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    Encoder encoder = new EncoderFactory().binaryEncoder(outputStream, null);
    datumWriter.write(user, encoder);
    encoder.flush();
    Event event = EventBuilder.withBody(outputStream.toByteArray());
    XContentBuilder expected = generateContentBuilder();
    XContentBuilder actual = avroSerializer.serialize(event);
    JsonParser parser = new JsonParser();
    assertEquals(parser.parse(expected.string()), parser.parse(actual.string()));
}
 

import org.apache.avro.io.DatumWriter; //导入依赖的package包/类
@Override
public byte[] serialize(String s, WrapperAppMessage wrapperAppMessage) {
    DatumWriter<WrapperAppMessage> datumWriter = new SpecificDatumWriter<>(wrapperAppMessage.getSchema());
    ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
    BinaryEncoder binaryEncoder = EncoderFactory.get().binaryEncoder(outputStream, null);
    try {
        datumWriter.write(wrapperAppMessage, binaryEncoder);
        binaryEncoder.flush();//带缓冲区的binaryEncoder和直接directBinaryEncoder不一样，需要flush一下，否则字节数组没有数据
        outputStream.flush();
        outputStream.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
    byte[] data = outputStream.toByteArray();
    return data;
}
 

import org.apache.avro.io.DatumWriter; //导入依赖的package包/类
/**
 * 动态序列化：通过动态解析Schema文件进行内容设置，并序列化内容
 * 
 * @throws IOException
 */
public void MemberInfoDynSer() throws IOException {
    // 1.解析schema文件内容
    Parser parser = new Parser();
    Schema mSchema = parser.parse(this.getClass().getResourceAsStream("/Members.avsc"));
    // 2.构建数据写对象
    DatumWriter<GenericRecord> mGr = new SpecificDatumWriter<GenericRecord>(mSchema);
    DataFileWriter<GenericRecord> mDfw = new DataFileWriter<GenericRecord>(mGr);
    // 3.创建序列化文件
    mDfw.create(mSchema, new File("/Users/a/Desktop/tmp/members.avro"));
    // 4.添加序列化数据
    for (int i = 0; i < 20; i++) {
        GenericRecord gr = new GenericData.Record(mSchema);
        int r = i * new Random().nextInt(50);
        gr.put("userName", "light-" + r);
        gr.put("userPwd", "2016-" + r);
        gr.put("realName", "滔滔" + r + "号");
        mDfw.append(gr);
    }
    // 5.关闭数据文件写对象
    mDfw.close();
    System.out.println("Dyn Builder Ser Start Complete.");
}
 

import org.apache.avro.io.DatumWriter; //导入依赖的package包/类
/**
 * Serialize the record to prepare for publishing.
 *
 * @param record the GenericRecord
 * @param schema the Avro Schema
 * @param ggAvroSchema the internal representation of the Avro schema
 * @return the serialized record
 * @throws IOException if there is a problem
 */
private byte[] serializeRecord(GenericRecord record, Schema schema,
                               @SuppressWarnings("unused") AvroSchema ggAvroSchema) throws IOException {
    byte[] rval;
    
    BinaryEncoder encoder = null;
    
    // serialize the record into a byte array
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    DatumWriter<GenericRecord> writer = new GenericDatumWriter<GenericRecord>(schema);
    encoder = EncoderFactory.get().directBinaryEncoder(out, encoder);
    writer.write(record, encoder);
    encoder.flush();
    rval = out.toByteArray();
    //out.close(); // noop in the Apache version, so not bothering
    return rval;
}