Java Decoder类代码示例

import org.apache.avro.io.Decoder; //导入依赖的package包/类
public void jsonReadWriteExample() throws IOException {
	Employee employee = Employee.newBuilder().setFirstName("Gaurav")
			.setLastName("Mazra").setSex(SEX.MALE).build();

	DatumWriter<Employee> employeeWriter = new SpecificDatumWriter<>(Employee.class);
	byte[] data;
	try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
		Encoder jsonEncoder = EncoderFactory.get().jsonEncoder(Employee.getClassSchema(), baos);
		employeeWriter.write(employee, jsonEncoder);
		jsonEncoder.flush();
		data = baos.toByteArray();
	}
	
	// serialized data
	System.out.println(new String(data));
	
	DatumReader<Employee> employeeReader = new SpecificDatumReader<>(Employee.class);
	Decoder decoder = DecoderFactory.get().jsonDecoder(Employee.getClassSchema(), new String(data));
	employee = employeeReader.read(null, decoder);
	//data after deserialization
	System.out.println(employee);
}
 

import org.apache.avro.io.Decoder; //导入依赖的package包/类
public void binaryReadWriteExample() throws IOException {
	Employee employee = Employee.newBuilder().setFirstName("Gaurav")
			.setLastName("Mazra").setSex(SEX.MALE).build();

	DatumWriter<Employee> employeeWriter = new SpecificDatumWriter<>(Employee.class);
	byte[] data;
	try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) {
		Encoder binaryEncoder = EncoderFactory.get().binaryEncoder(baos, null);
		employeeWriter.write(employee, binaryEncoder);
		binaryEncoder.flush();
		data = baos.toByteArray();
	}
	
	// serialized data
	System.out.println(data);
	
	DatumReader<Employee> employeeReader = new SpecificDatumReader<>(Employee.class);
	Decoder binaryDecoder = DecoderFactory.get().binaryDecoder(data, null);
	employee = employeeReader.read(null, binaryDecoder);
	//data after deserialization
	System.out.println(employee);
}
 

import org.apache.avro.io.Decoder; //导入依赖的package包/类
private void checkNumeric(String type, Object value) throws Exception {
	String def =
			"{\"type\":\"record\",\"name\":\"X\",\"fields\":"
					+"[{\"type\":\""+type+"\",\"name\":\"n\"}]}";
	Schema schema = Schema.parse(def);
	DatumReader<GenericRecord> reader =
			new GenericDatumReader<GenericRecord>(schema);
	
	String[] records = {"{\"n\":1}", "{\"n\":1.0}"};
	
	for (String record : records) {
		Decoder decoder = new ExtendedJsonDecoder(schema, record);
		GenericRecord r = reader.read(null, decoder);
		Assert.assertEquals(value, r.get("n"));
	}
}
 

import org.apache.avro.io.Decoder; //导入依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public T deserialize(String topic, byte[] data) {
	try {
		T result = null;

		if (data != null) {
			LOGGER.debug("data='{}'", DatatypeConverter.printHexBinary(data));

			DatumReader<GenericRecord> datumReader = new SpecificDatumReader<>(
					targetType.newInstance().getSchema());
			Decoder decoder = DecoderFactory.get().binaryDecoder(data, null);

			result = (T) datumReader.read(null, decoder);
			LOGGER.debug("deserialized data='{}'", result);
		}
		return result;
	} catch (Exception ex) {
		throw new SerializationException(
				"Can't deserialize data '" + Arrays.toString(data) + "' from topic '" + topic + "'", ex);
	}
}
 

import org.apache.avro.io.Decoder; //导入依赖的package包/类
/**
 * Converts the avro binary data to the json format
 */
@Override
public XContentBuilder serialize(Event event) {
    XContentBuilder builder = null;
    try {
        if (datumReader != null) {
            Decoder decoder = new DecoderFactory().binaryDecoder(event.getBody(), null);
            GenericRecord data = datumReader.read(null, decoder);
            logger.trace("Record in event " + data);
            XContentParser parser = XContentFactory
                    .xContent(XContentType.JSON)
                    .createParser(NamedXContentRegistry.EMPTY, data.toString());
            builder = jsonBuilder().copyCurrentStructure(parser);
            parser.close();
        } else {
            logger.error("Schema File is not configured");
        }
    } catch (IOException e) {
        logger.error("Exception in parsing avro format data but continuing serialization to process further records",
                e.getMessage(), e);
    }
    return builder;
}
 

import org.apache.avro.io.Decoder; //导入依赖的package包/类
public Object fromBytes(Schema schema, byte data[]) throws GoraException {
  Schema fromSchema = null;
  if (schema.getType() == Type.UNION) {
    try {
      Decoder decoder = DecoderFactory.get().binaryDecoder(data, null);
      int unionIndex = decoder.readIndex();
      List<Schema> possibleTypes = schema.getTypes();
      fromSchema = possibleTypes.get(unionIndex);
      Schema effectiveSchema = possibleTypes.get(unionIndex);
      if (effectiveSchema.getType() == Type.NULL) {
        decoder.readNull();
        return null;
      } else {
        data = decoder.readBytes(null).array();
      }
    } catch (IOException e) {
      LOG.error(e.getMessage());
      throw new GoraException("Error decoding union type: ", e);
    }
  } else {
    fromSchema = schema;
  }
  return fromBytes(encoder, fromSchema, data);
}
 

import org.apache.avro.io.Decoder; //导入依赖的package包/类
/**
 * Reads a boolean[] from input
 * @throws IOException
 */
public static boolean[] readBoolArray(Decoder in) throws IOException {

  int length = in.readInt();
  boolean[] boolArr = new boolean[length];

  int byteArrLength = (int)Math.ceil(length / 8.0);
  byte[] byteArr = new byte[byteArrLength];
  in.readFixed(byteArr);

  int arrIndex = 0;
  byte b = 0;
  for(int i=0; i < length; i++) {
    if(i % 8 == 0) {
      b = byteArr[arrIndex++];
    }
    boolArr[i] = (b & 0x01) > 0;
    b >>= 1;
  }
  return boolArr;
}
 

import org.apache.avro.io.Decoder; //导入依赖的package包/类
/**
 * Processes an Avro Blob containing a single message and with no embedded
 * schema. This is the pattern when Avro objects are passed over messaging
 * infrastructure such as Apache Kafka.
 * 
 * @param avroMessage
 *            The Blob that holds the single Avro message object
 * @param avroKey
 *            The Blob that holds the single Avro key object (if passed)
 * @param outStream
 *            The stream to which the JSON string must be submitted
 * @param outTuple
 *            The tuple holding the JSON string
 * @param messageSchema
 *            The schema of the Avro messsage object
 * @param keySchema
 *            The schema of the Avro key object
 * @throws Exception
 */
private void processAvroMessage(Blob avroMessage, Blob avroKey, StreamingOutput<OutputTuple> outStream,
		OutputTuple outTuple, Schema messageSchema, Schema keySchema) throws Exception {
	// Deserialize message
	GenericDatumReader<GenericRecord> consumer = new GenericDatumReader<GenericRecord>(messageSchema);
	ByteArrayInputStream consumedByteArray = new ByteArrayInputStream(avroMessage.getData());
	Decoder consumedDecoder = DecoderFactory.get().binaryDecoder(consumedByteArray, null);
	GenericRecord consumedDatum = consumer.read(null, consumedDecoder);
	if (LOGGER.isTraceEnabled())
		LOGGER.log(TraceLevel.TRACE, "JSON representation of Avro message: " + consumedDatum.toString());
	outTuple.setString(outputJsonMessage, consumedDatum.toString());
	// Deserialize key (if specified)
	if (avroKey != null) {
		consumer = new GenericDatumReader<GenericRecord>(keySchema);
		consumedByteArray = new ByteArrayInputStream(avroKey.getData());
		consumedDecoder = DecoderFactory.get().binaryDecoder(consumedByteArray, null);
		consumedDatum = consumer.read(null, consumedDecoder);
		if (LOGGER.isTraceEnabled())
			LOGGER.log(TraceLevel.TRACE, "JSON representation of Avro key: " + consumedDatum.toString());
		if (outputJsonKey != null)
			outTuple.setString(outputJsonKey, consumedDatum.toString());
	}
	// Submit new tuple to output port 0
	outStream.submit(outTuple);
}
 

import org.apache.avro.io.Decoder; //导入依赖的package包/类
@Override
public <T> T decode(byte[] bytes, Class<T> type) throws IOException {
	Assert.notNull(bytes, "'bytes' cannot be null");
	Assert.notNull(bytes, "Class can not be null");
	ByteBuffer buf = ByteBuffer.wrap(bytes);
	byte[] payload = new byte[bytes.length-4];
	Integer schemaId = buf.getInt();
	buf.get(payload);
	Schema schema = schemaRegistryClient.fetch(schemaId);
	DatumReader reader = getDatumReader(type,schema);
	Decoder decoder = DecoderFactory.get().binaryDecoder(payload,null);
	return (T) reader.read(null,decoder);
}
 

import org.apache.avro.io.Decoder; //导入依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public T deserialize(String topic, byte[] data) {
  try {
    T result = null;

    if (data != null) {
      LOGGER.debug("data='{}'", DatatypeConverter.printHexBinary(data));

      DatumReader<GenericRecord> datumReader =
          new SpecificDatumReader<>(targetType.newInstance().getSchema());
      Decoder decoder = DecoderFactory.get().binaryDecoder(data, null);

      result = (T) datumReader.read(null, decoder);
      LOGGER.debug("deserialized data='{}'", result);
    }
    return result;
  } catch (Exception ex) {
    throw new SerializationException(
        "Can't deserialize data '" + Arrays.toString(data) + "' from topic '" + topic + "'", ex);
  }
}
 

import org.apache.avro.io.Decoder; //导入依赖的package包/类
@Override
protected GenericRecord decodeRecord(MessageAndOffset messageAndOffset) throws SchemaNotFoundException, IOException {
  byte[] payload = getBytes(messageAndOffset.message().payload());
  if (payload[0] != KafkaAvroSchemaRegistry.MAGIC_BYTE) {
    throw new RuntimeException(String.format("Unknown magic byte for partition %s", this.getCurrentPartition()));
  }

  byte[] schemaIdByteArray = Arrays.copyOfRange(payload, 1, 1 + KafkaAvroSchemaRegistry.SCHEMA_ID_LENGTH_BYTE);
  String schemaId = Hex.encodeHexString(schemaIdByteArray);
  Schema schema = null;
  schema = this.schemaRegistry.getSchemaById(schemaId);
  reader.get().setSchema(schema);
  Decoder binaryDecoder =
      DecoderFactory.get().binaryDecoder(payload, 1 + KafkaAvroSchemaRegistry.SCHEMA_ID_LENGTH_BYTE,
          payload.length - 1 - KafkaAvroSchemaRegistry.SCHEMA_ID_LENGTH_BYTE, null);
  try {
    GenericRecord record = reader.get().read(null, binaryDecoder);
    record = AvroUtils.convertRecordSchema(record, this.schema.get());
    return record;
  } catch (IOException e) {
    LOG.error(String.format("Error during decoding record for partition %s: ", this.getCurrentPartition()));
    throw e;
  }
}
 

import org.apache.avro.io.Decoder; //导入依赖的package包/类
@Test
public void testEnums() throws Exception {
  Schema schema = Enums.SCHEMA$;

  String avroJson = "{\"enum1\": \"X\", \"enum2\": {\"test.Enum2\": \"A\"}, \"enum3\": {\"null\": null}, \"enum4\": [{\"test.Enum4\": \"SAT\"}, {\"test.Enum4\": \"SUN\"}]}}";
  Decoder decoder = DecoderFactory.get().jsonDecoder(schema, avroJson);
  GenericDatumReader<Record> reader = new GenericDatumReader<Record>(schema);
  Record record1 = reader.read(null, decoder);

  String mongoJson = "{\"enum1\": \"X\", \"enum2\": \"A\", \"enum3\": null, \"enum4\": [\"SAT\", \"SUN\"]}}";
  BSONObject object = (BSONObject) JSON.parse(mongoJson);
  Record record2 = RecordConverter.toRecord(schema, object, getClass().getClassLoader());

  assertThat(record2, is(record1));
  assertThat(AvroHelper.toSimpleJson(schema, record2), is(AvroHelper.toSimpleJson(schema, record1)));
}
 

import org.apache.avro.io.Decoder; //导入依赖的package包/类
@Test
public void testUnions() throws Exception {
  Schema schema = Unions.SCHEMA$;

  String avroJson = "{\"union1\": {\"int\": 1}, \"union2\": {\"test.Union2\": {\"union21\": {\"long\": 2}}}, \"union3\": {\"array\": [{\"boolean\": true}, {\"boolean\": false}, {\"null\": null}]}, \"union4\": {\"map\": {\"a\": {\"string\": \"A\"}, \"b\": {\"string\": \"B\"}, \"c\": {\"string\": \"C\"}}}, \"union5\": {\"null\": null}, \"union6\": {\"null\": null}}";
  Decoder decoder = DecoderFactory.get().jsonDecoder(schema, avroJson);
  GenericDatumReader<Record> reader = new GenericDatumReader<Record>(schema);
  Record record1 = reader.read(null, decoder);

  String mongoJson = "{\"union1\": 1, \"union2\": {\"union21\": 2}, \"union3\": [true, false, null], \"union4\": {\"a\": \"A\", \"b\": \"B\", \"c\": \"C\"}, \"union5\": null, \"union6\": null}";
  DBObject object = (DBObject) JSON.parse(mongoJson);
  Record record2 = RecordConverter.toRecord(schema, object, getClass().getClassLoader());

  assertThat(record2, is(record1));
  assertThat(AvroHelper.toSimpleJson(schema, record2), is(AvroHelper.toSimpleJson(schema, record1)));
}
 

import org.apache.avro.io.Decoder; //导入依赖的package包/类
/**
 * Convert the payload in the input record to a deserialized object with the latest schema
 *
 * @param inputRecord the input record
 * @return the schema'ed payload object
 */
protected P upConvertPayload(GenericRecord inputRecord) throws DataConversionException {
  try {
    Schema payloadSchema = getPayloadSchema(inputRecord);
    // Set writer schema
    latestPayloadReader.setSchema(payloadSchema);

    byte[] payloadBytes = getPayloadBytes(inputRecord);
    Decoder decoder = DecoderFactory.get().binaryDecoder(payloadBytes, null);

    // 'latestPayloadReader.read' will convert the record from 'payloadSchema' to the latest payload schema
    return latestPayloadReader.read(null, decoder);
  } catch (Exception e) {
    throw new DataConversionException(e);
  }
}
 

import org.apache.avro.io.Decoder; //导入依赖的package包/类
@Override
public Collection<Either<JobSpec, URI>> parseJobSpec(byte[] message)
    throws IOException {

  InputStream is = new ByteArrayInputStream(message);
  this.versionWriter.readSchemaVersioningInformation(new DataInputStream(is));

  Decoder decoder = DecoderFactory.get().binaryDecoder(is, this.decoder.get());
  try {
    T decodedMessage = this.reader.get().read(null, decoder);
    return parseJobSpec(decodedMessage);
  } catch (AvroRuntimeException | IOException exc) {
    this.messageParseFailures.mark();
    if (this.messageParseFailures.getFiveMinuteRate() < 1) {
      log.warn("Unable to decode input message.", exc);
    } else {
      log.warn("Unable to decode input message.");
    }
    return Lists.newArrayList();
  }
}