Java BinaryDecoder类代码示例

import org.apache.avro.io.BinaryDecoder; //导入依赖的package包/类
static void convertAvroToJson(InputStream inputStream, OutputStream outputStream, Schema schema)
        throws IOException {
    DatumReader<Object> reader = new GenericDatumReader<>(schema);
    DatumWriter<Object> writer = new GenericDatumWriter<>(schema);

    BinaryDecoder binaryDecoder = DecoderFactory.get().binaryDecoder(inputStream, null);

    JsonEncoder jsonEncoder = EncoderFactory.get().jsonEncoder(schema, outputStream, true);
    Object datum = null;
    while (!binaryDecoder.isEnd()) {
        datum = reader.read(datum, binaryDecoder);
        writer.write(datum, jsonEncoder);
        jsonEncoder.flush();
    }
    outputStream.flush();
}
 

import org.apache.avro.io.BinaryDecoder; //导入依赖的package包/类
/**
 * Process list.
 *
 * @param weathers the weathers
 * @throws Exception the exception
 */
public void processList(List<Weather> weathers) throws Exception {
	long before = System.currentTimeMillis();
	BinaryEncoder binaryEncoder = null;
	BinaryDecoder binaryDecoder = null;
	Weather weatherRead = null;
	for (Weather weather : weathers) {
		DatumWriter<Weather> datumWriterWeather = new SpecificDatumWriter<Weather>(Weather.getClassSchema());
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		byte[] byteData = null;
		try {
			binaryEncoder = EncoderFactory.get().binaryEncoder(baos, binaryEncoder);
			datumWriterWeather.write(weather, binaryEncoder);
			binaryEncoder.flush();
			byteData = baos.toByteArray();
		} finally {
			baos.close();
		}

		DatumReader<Weather> datumReaderWeather = new SpecificDatumReader<Weather>(Weather.getClassSchema());
		binaryDecoder = DecoderFactory.get().binaryDecoder(byteData, binaryDecoder);
		weatherRead = datumReaderWeather.read(weatherRead, binaryDecoder);
		// System.out.println("After Binary Read: " + weatherRead.toString());
	}
	System.out.println("size=" + weathers.size() + ", elapsed: " + (System.currentTimeMillis()-before));
}
 

import org.apache.avro.io.BinaryDecoder; //导入依赖的package包/类
/**
 * Change the schema of an Avro record.
 * @param record The Avro record whose schema is to be changed.
 * @param newSchema The target schema. It must be compatible as reader schema with record.getSchema() as writer schema.
 * @return a new Avro record with the new schema.
 * @throws IOException if conversion failed.
 */
public static GenericRecord convertRecordSchema(GenericRecord record, Schema newSchema) throws IOException {
  if (record.getSchema().equals(newSchema)) {
    return record;
  }

  if (checkReaderWriterCompatibility(newSchema, record.getSchema()).getType() != COMPATIBLE) {
    LOG.debug("Record schema not compatible with writer schema. Converting record schema to writer schema may fail.");
  }

  try {
    BinaryDecoder decoder = new DecoderFactory().binaryDecoder(recordToByteArray(record), null);
    DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>(record.getSchema(), newSchema);
    return reader.read(null, decoder);
  } catch (IOException e) {
    throw new IOException(
        String.format("Cannot convert avro record to new schema. Origianl schema = %s, new schema = %s",
            record.getSchema(), newSchema),
        e);
  }
}
 

import org.apache.avro.io.BinaryDecoder; //导入依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public JetstreamEvent decode(byte[] key, byte[] message) {
    ByteArrayInputStream stream = new ByteArrayInputStream(message);
    BinaryDecoder reusedDecoder = decoderHolder.get();
    BinaryDecoder decoder = DecoderFactory.get().directBinaryDecoder(stream, reusedDecoder);
    if (reusedDecoder == null) {
        decoderHolder.set(decoder);
    }
    Record object;
    try {
        object = reader.read(null, decoder);
        Map<String, Object> m = (Map<String, Object>) object.get(MAP_FIELD_NAME);
        return new JetstreamEvent(m);
    } catch (IOException e) {
        throw new IllegalArgumentException("Can not read the avro message", e);
    }
}
 

import org.apache.avro.io.BinaryDecoder; //导入依赖的package包/类
/**
 * Change the schema of an Avro record.
 * @param record The Avro record whose schema is to be changed.
 * @param newSchema The target schema. It must be compatible as reader schema with record.getSchema() as writer schema.
 * @return a new Avro record with the new schema.
 * @throws IOException if conversion failed.
 */
public static GenericRecord convertRecordSchema(GenericRecord record, Schema newSchema) throws IOException {
  if (record.getSchema().equals(newSchema)) {
    return record;
  }

  try {
    BinaryDecoder decoder = new DecoderFactory().binaryDecoder(recordToByteArray(record), null);
    DatumReader<GenericRecord> reader = new GenericDatumReader<>(record.getSchema(), newSchema);
    return reader.read(null, decoder);
  } catch (IOException e) {
    throw new IOException(
        String.format("Cannot convert avro record to new schema. Origianl schema = %s, new schema = %s",
            record.getSchema(), newSchema),
        e);
  }
}
 

import org.apache.avro.io.BinaryDecoder; //导入依赖的package包/类
public KafkaAvroJobMonitor(String topic, MutableJobCatalog catalog, Config config, Schema schema,
    SchemaVersionWriter<?> versionWriter) {
  super(topic, catalog, config);

  this.schema = schema;
  this.decoder = new ThreadLocal<BinaryDecoder>() {
    @Override
    protected BinaryDecoder initialValue() {
      InputStream dummyInputStream = new ByteArrayInputStream(new byte[0]);
      return DecoderFactory.get().binaryDecoder(dummyInputStream, null);
    }
  };
  this.reader = new ThreadLocal<SpecificDatumReader<T>>() {
    @Override
    protected SpecificDatumReader<T> initialValue() {
      return new SpecificDatumReader<>(KafkaAvroJobMonitor.this.schema);
    }
  };
  this.versionWriter = versionWriter;
}
 

import org.apache.avro.io.BinaryDecoder; //导入依赖的package包/类
public void handleMsg(BytesMessage msg) {
    BinaryDecoder binaryDecoder = null;
    User user = null;
    try {
        if (msg.getBodyLength() > 0) {
            byte[] byteArray = new byte[(int)msg.getBodyLength()];
        	msg.readBytes(byteArray);
        	ByteArrayInputStream stream = 
                new ByteArrayInputStream(byteArray);
        	binaryDecoder = avroDecoderFactory.binaryDecoder(stream, 
                binaryDecoder);
        	user = avroUserReader.read(user, binaryDecoder);
        	System.out.println("firstName: " + user.getFirstName());
        	System.out.println("lastName: " + user.getLastName());
        	System.out.println("City: " + user.getCity());
        	System.out.println("State: " + user.getState());
        }
    } catch (Exception e) {
        throw new RuntimeException("Error processing Avro message", e);
    }
}
 

import org.apache.avro.io.BinaryDecoder; //导入依赖的package包/类
public EventList deserialize(String project, String collection, SliceInput slice) throws IOException {
    String json = slice.readSlice(slice.readInt()).toStringUtf8();
    Schema schema = new Schema.Parser().parse(json);
    int records = slice.readInt();

    BinaryDecoder binaryDecoder = DecoderFactory.get().directBinaryDecoder(slice, null);

    List<SchemaField> fields = metastore.getCollection(project, collection);
    Schema avroSchema = AvroUtil.convertAvroSchema(fields);

    GenericDatumReader<GenericRecord> reader = new GenericDatumReader(schema, avroSchema);

    List<Event> list = new ArrayList<>(records);
    for (int i = 0; i < records; i++) {
        GenericRecord record = reader.read(null, binaryDecoder);
        list.add(new Event(project, collection, null, fields, record));
    }

    return new EventList(Event.EventContext.empty(), project, list);
}
 

import org.apache.avro.io.BinaryDecoder; //导入依赖的package包/类
private void parseTopics() {
  if (state.getProperty(TOPIC_LIST) != null) {
    byte[] data = base64.decodeBase64(state.getProperty(TOPIC_LIST));
    BinaryDecoder decoder = DecoderFactory.get().binaryDecoder(data, null);
    SpecificDatumReader<Topic> avroReader = new SpecificDatumReader<>(Topic.class);
    try { // NOSONAR
      Topic decodedTopic;
      while (!decoder.isEnd()) {
        decodedTopic = avroReader.read(null, decoder);
        LOG.debug("Loaded {}", decodedTopic);
        topicMap.put(decodedTopic.getId(), decodedTopic);
      }
    } catch (Exception ex) {
      LOG.error("Unexpected exception occurred while reading information from decoder", ex);
    }
  } else {
    LOG.info("No topic list found in state");
  }
}
 

import org.apache.avro.io.BinaryDecoder; //导入依赖的package包/类
@Override
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
  int size = in.readInt();
  byte[] data = new byte[size];
  in.read(data);
  @SuppressWarnings("unchecked")
  AvroReader<T> reader = (AvroReader<T>) recordReaderMap.get().get(className);
  if (reader == null) {
    reader = new AvroReader<T>(
            new SpecificDatumReader<T>(clazz),
            DecoderFactory.get().binaryDecoder(data, null)
    );
    recordReaderMap.get().put(className, reader);
  }
  BinaryDecoder recordDataDecoder = DecoderFactory.get().binaryDecoder(data, reader.getDecoder());
  avroObject = reader.getReader().read(null, recordDataDecoder);
}
 

import org.apache.avro.io.BinaryDecoder; //导入依赖的package包/类
/**
 * Decode records from <code>ByteBuffer</code> and return <code>Iterable</code>.
 *
 * @param bb the <code>ByteBuffer</code> which decode
 * @return decoded <code>Iterable</code>
 */
public Iterable<T> decodeRecords(ByteBuffer bb) throws IOException {
  byte[] recordDataBody = toByteArray(bb);
  BinaryDecoder recordDataDecoder = DecoderFactory.get().binaryDecoder(recordDataBody, null);
  RecordData recordData = recordDataReader.read(null, recordDataDecoder);

  LOG.debug("Avro event header: {}", recordData.getRecordHeader());
  LOG.debug("Avro event data.size: {}", recordData.getEventRecords().size());

  List<T> results = new ArrayList<T>();
  BinaryDecoder recordDecoder = null;
  for (ByteBuffer eventBuf : recordData.getEventRecords()) {
    byte[] recordBody = toByteArray(eventBuf);
    recordDecoder = DecoderFactory.get().binaryDecoder(recordBody, recordDecoder);
    T record = recordReader.read(null, recordDecoder);
    results.add(record);
    LOG.trace("Parsed record: {}", record);
  }
  return results;
}
 

import org.apache.avro.io.BinaryDecoder; //导入依赖的package包/类
/**
 * Read tokens from a file and add them to the user's credentials.
 * @param ugi user's credentials to add tokens to.
 * @throws IOException if there are errors in reading the tokens' file.
 */
void addTokensFromFile(final UserGroupInformation ugi) throws IOException {
  LOG.log(Level.FINE, "Reading security tokens from file: {0}", this.securityTokensFile);

  try (final FileInputStream stream = new FileInputStream(securityTokensFile)) {
    final BinaryDecoder decoder = decoderFactory.binaryDecoder(stream, null);

    while (!decoder.isEnd()) {
      final SecurityToken token = tokenDatumReader.read(null, decoder);

      final Token<TokenIdentifier> yarnToken = new Token<>(
          token.getKey().array(),
          token.getPassword().array(),
          new Text(token.getKind().toString()),
          new Text(token.getService().toString()));

      LOG.log(Level.FINE, "addToken for {0}", yarnToken.getKind());

      ugi.addToken(yarnToken);
    }
  }
}
 

import org.apache.avro.io.BinaryDecoder; //导入依赖的package包/类
/***
 * Deserialize byte stream into an AvroSimpleFeature
 *
 * @param avroData
 *            serialized bytes of AvroSimpleFeature
 * @param avroObjectToReuse
 *            null or AvroSimpleFeature instance to be re-used. If null a
 *            new object will be allocated.
 * @return instance of AvroSimpleFeature with values parsed from avroData
 * @throws IOException
 */
private static AvroSimpleFeature deserializeASF(
		final byte[] avroData,
		AvroSimpleFeature avroObjectToReuse )
		throws IOException {
	final BinaryDecoder decoder = DECODER_FACTORY.binaryDecoder(
			avroData,
			null);
	if (avroObjectToReuse == null) {
		avroObjectToReuse = new AvroSimpleFeature();
	}

	DATUM_READER.setSchema(avroObjectToReuse.getSchema());
	return DATUM_READER.read(
			avroObjectToReuse,
			decoder);
}
 

import org.apache.avro.io.BinaryDecoder; //导入依赖的package包/类
private static <T> T deserialize(
		final T avroObject,
		final byte[] avroData,
		Class<T> avroClass,
		Schema avroSchema )
		throws IOException {
	BinaryDecoder decoder = df.binaryDecoder(
			avroData,
			null);
	if (!readers.containsKey(avroClass.toString())) {
		readers.put(
				avroClass.toString(),
				new SpecificDatumReader(
						avroSchema));
	}
	SpecificDatumReader<T> reader = readers.get(avroClass.toString());
	return reader.read(
			avroObject,
			decoder);
}
 

import org.apache.avro.io.BinaryDecoder; //导入依赖的package包/类
@Override
public int compare(byte[] o1, int s1, int l1, byte[] o2, int s2, int l2) {
  try {
    final BinaryDecoder d1 = DECODER_FACTORY.binaryDecoder(o1, s1, l1, null);
    final ByteBuffer key1 = d1.readBytes(null);

    // re-use the decoder instance, but do not re-use the byte buffer,
    // because DecoratedKey stores a reference
    final BinaryDecoder d2 = DECODER_FACTORY.binaryDecoder(o2, s2, l2, d1);
    final ByteBuffer key2 = d2.readBytes(null);

    return compare(key1, key2);
  } catch (final IOException e) {
    throw Throwables.propagate(e);
  }
}