Java ReflectData类代码示例

import org.apache.avro.reflect.ReflectData; //导入依赖的package包/类
/**
 * Creates a flat Avro Schema for testing.
 */
public static Schema createFlatAvroSchema(String[] fieldNames, TypeInformation[] fieldTypes) {
	final SchemaBuilder.FieldAssembler<Schema> fieldAssembler = SchemaBuilder
		.record("BasicAvroRecord")
		.namespace(NAMESPACE)
		.fields();

	final Schema nullSchema = Schema.create(Schema.Type.NULL);

	for (int i = 0; i < fieldNames.length; i++) {
		Schema schema = ReflectData.get().getSchema(fieldTypes[i].getTypeClass());
		Schema unionSchema = Schema.createUnion(Arrays.asList(nullSchema, schema));
		fieldAssembler.name(fieldNames[i]).type(unionSchema).noDefault();
	}

	return fieldAssembler.endRecord();
}
 

import org.apache.avro.reflect.ReflectData; //导入依赖的package包/类
public static void testReflect(Object value, Type type, String schema)
  throws Exception {

  // check that schema matches expected
  Schema s = ReflectData.get().getSchema(type);
  assertEquals(Schema.parse(schema), s);

  // check that value is serialized correctly
  ReflectDatumWriter<Object> writer = new ReflectDatumWriter<Object>(s);
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  writer.write(value, EncoderFactory.get().directBinaryEncoder(out, null));
  ReflectDatumReader<Object> reader = new ReflectDatumReader<Object>(s);
  Object after =
    reader.read(null,
                DecoderFactory.get().binaryDecoder(out.toByteArray(), null));
  assertEquals(value, after);
}
 

import org.apache.avro.reflect.ReflectData; //导入依赖的package包/类
public static List<IndividualWrapper<Individual, FitnessValue>> readIndividualsFromFile(
        Path filePath, Configuration configuration) throws IOException {
    List<IndividualWrapper<Individual, FitnessValue>> result =
            new ArrayList<IndividualWrapper<Individual, FitnessValue>>();

    SeekableInput seekableFileInput = new FsInput(filePath, configuration);
    ReflectData reflectData = new ReflectData(configuration.getClassLoader());
    DatumReader<IndividualWrapper<Individual, FitnessValue>> datumReader = new ReflectDatumReader<IndividualWrapper<Individual, FitnessValue>>(reflectData);
    DataFileReader<IndividualWrapper<Individual, FitnessValue>> avroFileReader =
            new DataFileReader<IndividualWrapper<Individual, FitnessValue>>(seekableFileInput, datumReader);

    for (IndividualWrapper<Individual, FitnessValue> individualWrapper : avroFileReader)
        result.add(individualWrapper);

    avroFileReader.close();
    return result;
}
 

import org.apache.avro.reflect.ReflectData; //导入依赖的package包/类
@SuppressWarnings("unchecked")
public static <T> T fromBytes( byte[] val, Schema schema
    , SpecificDatumReader<T> datumReader, T object)
throws IOException {
  Type type = schema.getType();
  switch (type) {
  case ENUM:
    String symbol = schema.getEnumSymbols().get(val[0]);
    return (T)Enum.valueOf(ReflectData.get().getClass(schema), symbol);
  case STRING:  return (T)new Utf8(toString(val));
  case BYTES:   return (T)ByteBuffer.wrap(val);
  case INT:     return (T)Integer.valueOf(bytesToVint(val));
  case LONG:    return (T)Long.valueOf(bytesToVlong(val));
  case FLOAT:   return (T)Float.valueOf(toFloat(val));
  case DOUBLE:  return (T)Double.valueOf(toDouble(val));
  case BOOLEAN: return (T)Boolean.valueOf(val[0] != 0);
  case RECORD:  //fall
  case MAP:
  case ARRAY:   return (T)IOUtils.deserialize(val, (SpecificDatumReader<SpecificRecord>)datumReader, schema, (SpecificRecord)object);
  default: throw new RuntimeException("Unknown type: "+type);
  }
}
 

import org.apache.avro.reflect.ReflectData; //导入依赖的package包/类
private Schema  getSchema(Object payload){
	Schema schema = null;
	logger.debug("Obtaining schema for class {}", payload.getClass());
	if(GenericContainer.class.isAssignableFrom(payload.getClass())) {
		schema = ((GenericContainer) payload).getSchema();
		logger.debug("Avro type detected, using schema from object");
	}else{
		Integer id = localSchemaMap.get(payload.getClass().getName());
		if(id == null){
			if(!properties.isDynamicSchemaGenerationEnabled()) {
				throw new SchemaNotFoundException(String.format("No schema found on local cache for %s", payload.getClass()));
			}
			else{
				Schema localSchema = ReflectData.get().getSchema(payload.getClass());
				id = schemaRegistryClient.register(localSchema);
			}

		}
		schema = schemaRegistryClient.fetch(id);
	}

	return schema;
}
 

import org.apache.avro.reflect.ReflectData; //导入依赖的package包/类
@Test
public void dynamicReflectEncoderReflectDecoder() throws Exception{
	Schema schema = ReflectData.get().getSchema(Account.class);
	SchemaRegistryClient client = mock(SchemaRegistryClient.class);
	when(client.register(any())).thenReturn(10);
	when(client.fetch(eq(10))).thenReturn(schema);
	AvroCodec codec = new AvroCodec();
	AvroCodecProperties properties = new AvroCodecProperties();
	properties.setDynamicSchemaGenerationEnabled(true);
	codec.setProperties(properties);
	codec.setSchemaRegistryClient(client);
	codec.setResolver(new PathMatchingResourcePatternResolver(new AnnotationConfigApplicationContext()));
	codec.init();
	Account account = new Account();
	account.setCreatedAt(System.currentTimeMillis());
	account.setId(1L);
	byte[] results = codec.encode(account);
	Account decoded = codec.decode(results,Account.class);
	Assert.assertEquals(account.getId(), decoded.getId());
}
 

import org.apache.avro.reflect.ReflectData; //导入依赖的package包/类
@Test
public void testCreationWithSchema() throws Exception {
  List<Bird> expected = createRandomRecords(100);
  String filename = generateTestFile("tmp.avro", expected, SyncBehavior.SYNC_DEFAULT, 0,
      AvroCoder.of(Bird.class), DataFileConstants.NULL_CODEC);

  // Create a source with a schema object
  Schema schema = ReflectData.get().getSchema(Bird.class);
  AvroSource<GenericRecord> source = AvroSource.from(filename).withSchema(schema);
  List<GenericRecord> records = SourceTestUtils.readFromSource(source, null);
  assertEqualsWithGeneric(expected, records);

  // Create a source with a JSON schema
  String schemaString = ReflectData.get().getSchema(Bird.class).toString();
  source = AvroSource.from(filename).withSchema(schemaString);
  records = SourceTestUtils.readFromSource(source, null);
  assertEqualsWithGeneric(expected, records);
}
 

import org.apache.avro.reflect.ReflectData; //导入依赖的package包/类
private void initializeAvro() {
	final ClassLoader cl = Thread.currentThread().getContextClassLoader();

	if (SpecificRecord.class.isAssignableFrom(type)) {
		this.avroData = new SpecificData(cl);
		this.schema = this.avroData.getSchema(type);
		this.reader = new SpecificDatumReader<>(schema, schema, avroData);
		this.writer = new SpecificDatumWriter<>(schema, avroData);
	}
	else {
		final ReflectData reflectData = new ReflectData(cl);
		this.avroData = reflectData;
		this.schema = this.avroData.getSchema(type);
		this.reader = new ReflectDatumReader<>(schema, schema, reflectData);
		this.writer = new ReflectDatumWriter<>(schema, reflectData);
	}

	this.encoder = new DataOutputEncoder();
	this.decoder = new DataInputDecoder();
}
 

import org.apache.avro.reflect.ReflectData; //导入依赖的package包/类
@Test
public void testWriteAVROFileGenericRecord() {

    try {
        //Creation of EnrichedEventBody
        EnrichedEventBody enrichedEventBody = createEnrichedEventBody();

        //Get schema from EnrichedEventBody class
        Schema schema = ReflectData.get().getSchema(EnrichedEventBody.class);

        //Transform EnrichedEventBody object to GenericRecord object
        GenericRecord genericRecord = SerializationUtils.enrichedEventBody2GenericRecord(enrichedEventBody, schema);

        //Serialization of GenericRecord object into a AVRO file
        File fileAvro = new File("fileAvro1.avro");
        SerializationUtils.writeAVROFile(fileAvro, schema, genericRecord);


    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("Unexpected exception in testWriteAVROFileGenericRecord");
    }

}
 

import org.apache.avro.reflect.ReflectData; //导入依赖的package包/类
@Test
public void testWriteAVROFileGenericRecordCodec() {

    try {
        //Creation of EnrichedEventBody
        EnrichedEventBody enrichedEventBody = createEnrichedEventBody();

        //Get schema from EnrichedEventBody class
        Schema schema = ReflectData.get().getSchema(EnrichedEventBody.class);

        //Transform EnrichedEventBody object to GenericRecord object
        GenericRecord genericRecord = SerializationUtils.enrichedEventBody2GenericRecord(enrichedEventBody, schema);

        //Serialization of GenericRecord object into a AVRO file (with snappy codec)
        File fileAvro = new File("fileAvro2.avro");
        SerializationUtils.writeAVROFile(fileAvro, schema, genericRecord, CodecFactory.snappyCodec());

    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail("Unexpected exception in testWriteAVROFileGenericRecordCodec");
    }

}
 

import org.apache.avro.reflect.ReflectData; //导入依赖的package包/类
@Override
public byte[] serialise(final Object object) throws SerialisationException {
    Schema schema = ReflectData.get().getSchema(object.getClass());
    DatumWriter<Object> datumWriter = new ReflectDatumWriter<>(schema);
    DataFileWriter<Object> dataFileWriter = new DataFileWriter<>(datumWriter);
    ByteArrayOutputStream byteOut = new ByteArrayOutputStream();
    try {
        dataFileWriter.create(schema, byteOut);
        dataFileWriter.append(object);
        dataFileWriter.flush();
    } catch (final IOException e) {
        throw new SerialisationException("Unable to serialise given object of class: " + object.getClass().getName(), e);
    } finally {
        close(dataFileWriter);
    }
    return byteOut.toByteArray();
}
 

import org.apache.avro.reflect.ReflectData; //导入依赖的package包/类
private void init(String path) throws Exception {
        Configuration conf = new Configuration();
        conf.set("fs.defaultFS", "hdfs://localhost:8020");
        DatasetRepository datasetRepository = new FileSystemDatasetRepository.Builder()
                .rootDirectory(new URI(path)).configuration(conf).build();
        Schema schema = ReflectData.get().getSchema(FileInfo.class);
//        PartitionStrategy partitionStrategy = new PartitionStrategy.Builder().hash("pkey", "HASH", 5).build();
        PartitionStrategy partitionStrategy = new PartitionStrategy.Builder().identity("pkey", "ID", 10).build();
        DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
                .schema(schema)
                .format(Formats.AVRO)
                .partitionStrategy(partitionStrategy)
                .build();
        Dataset<FileInfo> dataset = datasetRepository.create("fileinfo", descriptor);
        writer = dataset.newWriter();
        System.out.println("** USING: " + writer);
        writer.open();
    }
 

import org.apache.avro.reflect.ReflectData; //导入依赖的package包/类
private Path writePojosToParquetFile( int num, CompressionCodecName compression,
                                      boolean enableDictionary) throws IOException {
  File tmp = File.createTempFile(getClass().getSimpleName(), ".tmp");
  tmp.deleteOnExit();
  tmp.delete();
  Path path = new Path(tmp.getPath());

  Pojo object = getPojo();

  Schema schema = ReflectData.get().getSchema(object.getClass());
  ParquetWriter<Pojo> writer = AvroParquetWriter.<Pojo>builder(path)
      .withSchema(schema)
      .withCompressionCodec(compression)
      .withDataModel(ReflectData.get())
      .withDictionaryEncoding(enableDictionary)
      .build();
  for (int i = 0; i < num; i++) {
    writer.write(object);
  }
  writer.close();
  return path;
}
 

import org.apache.avro.reflect.ReflectData; //导入依赖的package包/类
@Test(expected = ClassCastException.class)
public void testWriteUUIDMissingLogicalType() throws IOException {
  Schema uuidSchema = SchemaBuilder.record(RecordWithUUID.class.getName())
      .fields().requiredString("uuid").endRecord();
  LogicalTypes.uuid().addToSchema(uuidSchema.getField("uuid").schema());

  UUID u1 = UUID.randomUUID();
  UUID u2 = UUID.randomUUID();

  RecordWithUUID r1 = new RecordWithUUID();
  r1.uuid = u1;
  RecordWithUUID r2 = new RecordWithUUID();
  r2.uuid = u2;

  // write without using REFLECT, which has the logical type
  File test = write(uuidSchema, r1, r2);

  // verify that the field's type overrides the logical type
  Schema uuidStringSchema = SchemaBuilder
      .record(RecordWithStringUUID.class.getName())
      .fields().requiredString("uuid").endRecord();

  // this fails with an AppendWriteException wrapping ClassCastException
  // because the UUID isn't converted to a CharSequence expected internally
  read(ReflectData.get(), uuidStringSchema, test);
}
 

import org.apache.avro.reflect.ReflectData; //导入依赖的package包/类
@SuppressWarnings("unchecked")
public static <T> T fromBytes( byte[] val, Schema schema
    , SpecificDatumReader<T> datumReader, T object)
throws IOException {
  Type type = schema.getType();
  switch (type) {
  case ENUM:
    String symbol = schema.getEnumSymbols().get(val[0]);
    return (T)Enum.valueOf(ReflectData.get().getClass(schema), symbol);
  case STRING:  return (T)new Utf8(toString(val));
  case BYTES:   return (T)ByteBuffer.wrap(val);
  case INT:     return (T)Integer.valueOf(bytesToVint(val));
  case LONG:    return (T)Long.valueOf(bytesToVlong(val));
  case FLOAT:   return (T)Float.valueOf(toFloat(val));
  case DOUBLE:  return (T)Double.valueOf(toDouble(val));
  case BOOLEAN: return (T)Boolean.valueOf(val[0] != 0);
  case RECORD:  //fall
  case MAP:
  case ARRAY:   return (T)IOUtils.deserialize(val, (SpecificDatumReader<SpecificRecord>)datumReader, (SpecificRecord)object);
  default: throw new RuntimeException("Unknown type: "+type);
  }
}