Java ReflectDatumReader类代码示例

import org.apache.avro.reflect.ReflectDatumReader; //导入依赖的package包/类
public static void testReflect(Object value, Type type, String schema)
  throws Exception {

  // check that schema matches expected
  Schema s = ReflectData.get().getSchema(type);
  assertEquals(Schema.parse(schema), s);

  // check that value is serialized correctly
  ReflectDatumWriter<Object> writer = new ReflectDatumWriter<Object>(s);
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  writer.write(value, EncoderFactory.get().directBinaryEncoder(out, null));
  ReflectDatumReader<Object> reader = new ReflectDatumReader<Object>(s);
  Object after =
    reader.read(null,
                DecoderFactory.get().binaryDecoder(out.toByteArray(), null));
  assertEquals(value, after);
}
 

import org.apache.avro.reflect.ReflectDatumReader; //导入依赖的package包/类
public static List<IndividualWrapper<Individual, FitnessValue>> readIndividualsFromFile(
        Path filePath, Configuration configuration) throws IOException {
    List<IndividualWrapper<Individual, FitnessValue>> result =
            new ArrayList<IndividualWrapper<Individual, FitnessValue>>();

    SeekableInput seekableFileInput = new FsInput(filePath, configuration);
    ReflectData reflectData = new ReflectData(configuration.getClassLoader());
    DatumReader<IndividualWrapper<Individual, FitnessValue>> datumReader = new ReflectDatumReader<IndividualWrapper<Individual, FitnessValue>>(reflectData);
    DataFileReader<IndividualWrapper<Individual, FitnessValue>> avroFileReader =
            new DataFileReader<IndividualWrapper<Individual, FitnessValue>>(seekableFileInput, datumReader);

    for (IndividualWrapper<Individual, FitnessValue> individualWrapper : avroFileReader)
        result.add(individualWrapper);

    avroFileReader.close();
    return result;
}
 

import org.apache.avro.reflect.ReflectDatumReader; //导入依赖的package包/类
AvroBlock(
    byte[] data,
    long numRecords,
    Mode<T> mode,
    String writerSchemaString,
    String codec)
    throws IOException {
  this.mode = mode;
  this.numRecords = numRecords;
  checkNotNull(writerSchemaString, "writerSchemaString");
  Schema writerSchema = internOrParseSchemaString(writerSchemaString);
  Schema readerSchema =
      internOrParseSchemaString(
          MoreObjects.firstNonNull(mode.readerSchemaString, writerSchemaString));
  this.reader =
      (mode.type == GenericRecord.class)
          ? new GenericDatumReader<T>(writerSchema, readerSchema)
          : new ReflectDatumReader<T>(writerSchema, readerSchema);
  this.decoder = DecoderFactory.get().binaryDecoder(decodeAsInputStream(data, codec), null);
}
 

import org.apache.avro.reflect.ReflectDatumReader; //导入依赖的package包/类
private DataFileReader<E> initReader(FileInputSplit split) throws IOException {
	DatumReader<E> datumReader;

	if (org.apache.avro.generic.GenericRecord.class == avroValueType) {
		datumReader = new GenericDatumReader<E>();
	} else {
		datumReader = org.apache.avro.specific.SpecificRecordBase.class.isAssignableFrom(avroValueType)
			? new SpecificDatumReader<E>(avroValueType) : new ReflectDatumReader<E>(avroValueType);
	}
	if (LOG.isInfoEnabled()) {
		LOG.info("Opening split {}", split);
	}

	SeekableInput in = new FSDataInputStreamWrapper(stream, split.getPath().getFileSystem().getFileStatus(split.getPath()).getLen());
	DataFileReader<E> dataFileReader = (DataFileReader) DataFileReader.openReader(in, datumReader);

	if (LOG.isDebugEnabled()) {
		LOG.debug("Loaded SCHEMA: {}", dataFileReader.getSchema());
	}

	end = split.getStart() + split.getLength();
	recordsReadSinceLastSync = 0;
	return dataFileReader;
}
 

import org.apache.avro.reflect.ReflectDatumReader; //导入依赖的package包/类
private void initializeAvro() {
	final ClassLoader cl = Thread.currentThread().getContextClassLoader();

	if (SpecificRecord.class.isAssignableFrom(type)) {
		this.avroData = new SpecificData(cl);
		this.schema = this.avroData.getSchema(type);
		this.reader = new SpecificDatumReader<>(schema, schema, avroData);
		this.writer = new SpecificDatumWriter<>(schema, avroData);
	}
	else {
		final ReflectData reflectData = new ReflectData(cl);
		this.avroData = reflectData;
		this.schema = this.avroData.getSchema(type);
		this.reader = new ReflectDatumReader<>(schema, schema, reflectData);
		this.writer = new ReflectDatumWriter<>(schema, reflectData);
	}

	this.encoder = new DataOutputEncoder();
	this.decoder = new DataInputDecoder();
}
 

import org.apache.avro.reflect.ReflectDatumReader; //导入依赖的package包/类
public static Object deserializeFromByte(byte[] byteArray)
		throws IOException {
	SeekableByteArrayInput sin = new SeekableByteArrayInput(byteArray);
	System.out.println("length of read input stream " + sin.length());
	DatumReader<?> reader2 = new ReflectDatumReader<>();
	DataFileReader<?> in = new DataFileReader<>(sin, reader2);
	System.out.println(in.getSchema());
	System.out.println(in.hasNext());
	Object returnObject = null;
	System.out.println(in.getSchema().getFullName() );
	while (in.hasNext()) {
		returnObject = in.next();
		
	}
	in.close();
	return returnObject;
}
 

import org.apache.avro.reflect.ReflectDatumReader; //导入依赖的package包/类
@Override
public void open(FileInputSplit split) throws IOException {
	super.open(split);

	DatumReader<E> datumReader;
	if (org.apache.avro.specific.SpecificRecordBase.class.isAssignableFrom(avroValueType)) {
		datumReader = new SpecificDatumReader<E>(avroValueType);
	} else {
		datumReader = new ReflectDatumReader<E>(avroValueType);
	}
	
	LOG.info("Opening split " + split);
	
	SeekableInput in = new FSDataInputStreamWrapper(stream, (int) split.getLength());
	
	dataFileReader = DataFileReader.openReader(in, datumReader);
	dataFileReader.sync(split.getStart());
}
 

import org.apache.avro.reflect.ReflectDatumReader; //导入依赖的package包/类
@Override
public void open() {
  Preconditions.checkState(state.equals(ReaderWriterState.NEW),
    "A reader may not be opened more than once - current state:%s", state);

  logger.debug("Opening reader on path:{}", path);

  try {
    reader = new DataFileReader<E>(new AvroFSInput(fileSystem.open(path),
      fileSystem.getFileStatus(path).getLen()), new ReflectDatumReader<E>(
      schema));
  } catch (IOException e) {
    throw new DatasetReaderException("Unable to create reader path:" + path, e);
  }

  state = ReaderWriterState.OPEN;
}
 

import org.apache.avro.reflect.ReflectDatumReader; //导入依赖的package包/类
public List<Object> writeAvroFilesToStdout(Job job, Path outputPath) throws IOException {
    List<Object> records = new ArrayList<Object>();
    // Check that the results from the MapReduce were as expected.
    FileSystem fileSystem = FileSystem.get(job.getConfiguration());
    FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/part-*"));
    Assert.assertEquals(1, outputFiles.length);
    DataFileReader<Object> reader = new DataFileReader<Object>(
            new FsInput(outputFiles[0].getPath(), job.getConfiguration()),
            new ReflectDatumReader<Object>());
    for (Object record : reader) {
        records.add(record);
        System.out.println(record);
    }
    reader.close();
    return records;
}
 

import org.apache.avro.reflect.ReflectDatumReader; //导入依赖的package包/类
public void assertOutputResults(Job job, Path outputPath, Object[] expectedOutputs) throws IOException {
    List<Object> records = new ArrayList<Object>();
    // Check that the results from the MapReduce were as expected.
    FileSystem fileSystem = FileSystem.get(job.getConfiguration());
    FileStatus[] outputFiles = fileSystem.globStatus(outputPath.suffix("/part-*"));
    Assert.assertEquals(1, outputFiles.length);
    DataFileReader<Object> reader = new DataFileReader<Object>(
            new FsInput(outputFiles[0].getPath(), job.getConfiguration()),
            new ReflectDatumReader<Object>());
    for (Object record : reader) {
        records.add(record);
        System.out.println(record);
    }
    reader.close();

    assertArrayEquals(expectedOutputs, records.toArray());
}
 

import org.apache.avro.reflect.ReflectDatumReader; //导入依赖的package包/类
@InterfaceAudience.Private
@Override
public DatumReader getReader(Class<Object> clazz) {
  try {
    return new ReflectDatumReader(clazz);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

import org.apache.avro.reflect.ReflectDatumReader; //导入依赖的package包/类
/**
 * Reads the number of individuals into a file from meta-data.
 *
 * @param filePath      the file path
 * @param configuration the configuration
 * @return the number of individuals
 * @throws IOException
 */
public static int readNumberOfIndividuals(Path filePath, Configuration configuration) throws IOException {
    SeekableInput seekableFileInput = new FsInput(filePath, configuration);
    ReflectData reflectData = new ReflectData(configuration.getClassLoader());
    DatumReader<IndividualWrapper<Individual, FitnessValue>> datumReader =
            new ReflectDatumReader<IndividualWrapper<Individual, FitnessValue>>(reflectData);
    DataFileReader<IndividualWrapper<Individual, FitnessValue>> avroFileReader =
            new DataFileReader<IndividualWrapper<Individual, FitnessValue>>(seekableFileInput, datumReader);

    int result = (int) avroFileReader.getMetaLong(Constants.AVRO_NUMBER_OF_RECORDS);

    avroFileReader.close();

    return result;
}
 

import org.apache.avro.reflect.ReflectDatumReader; //导入依赖的package包/类
/**
 * Initialises the reader.
 */
@Override
public void initialize(InputSplit split, TaskAttemptContext context)
        throws IOException, InterruptedException {

    // Checks if the split is compatible.
    if (!(split instanceof PopulationInputSplit))
        throw new IllegalArgumentException("Only compatible with PopulationInputSplits.");

    PopulationInputSplit populationSplit = (PopulationInputSplit) split;

    // Prepares the fields.
    this.currentSplit = populationSplit;
    this.configuration = context.getConfiguration();
    this.currentRecordNumber = 0;
    this.currentDatum = null;
    this.currentKey = new AvroKey<Type>(null);
    this.currentValue = new IntWritable();

    // Initialises the file reader.
    if (!populationSplit.isInitialisationActive()) {
        SeekableInput seekableFileInput = new FsInput(this.currentSplit.getFilePath(), this.configuration);
        ReflectData reflectData = new ReflectData(this.configuration.getClassLoader());
        DatumReader<Type> datumReader = new ReflectDatumReader<Type>(reflectData);
        this.avroFileReader = new DataFileReader<Type>(seekableFileInput, datumReader);
    }

    // Initialises the partitioner nodes destinations list.
    this.partitionerNodesDestinations = populationSplit.getPartitionerNodesDestinations();
}
 

import org.apache.avro.reflect.ReflectDatumReader; //导入依赖的package包/类
private DatumReader getDatumReader(Class<?> type, Schema writer){
	DatumReader reader = null;
	if(SpecificRecord.class.isAssignableFrom(type)){
		reader = new SpecificDatumReader<>(writer,getReaderSchema(writer));
	}
	else if(GenericRecord.class.isAssignableFrom(type)){
		reader = new GenericDatumReader<>(writer,getReaderSchema(writer));
	}else{
		reader = new ReflectDatumReader<>(writer,getReaderSchema(writer));
	}

	return reader;
}
 

import org.apache.avro.reflect.ReflectDatumReader; //导入依赖的package包/类
protected AvroCoder(Class<T> type, Schema schema) {
  this.type = type;
  this.schemaSupplier = new SerializableSchemaSupplier(schema);
  typeDescriptor = TypeDescriptor.of(type);
  nonDeterministicReasons = new AvroDeterminismChecker().check(TypeDescriptor.of(type), schema);

  // Decoder and Encoder start off null for each thread. They are allocated and potentially
  // reused inside encode/decode.
  this.decoder = new EmptyOnDeserializationThreadLocal<>();
  this.encoder = new EmptyOnDeserializationThreadLocal<>();

  this.reflectData = Suppliers.memoize(new SerializableReflectDataSupplier(getType()));

  // Reader and writer are allocated once per thread per Coder
  this.reader =
      new EmptyOnDeserializationThreadLocal<DatumReader<T>>() {
        private final AvroCoder<T> myCoder = AvroCoder.this;

        @Override
        public DatumReader<T> initialValue() {
          return myCoder.getType().equals(GenericRecord.class)
              ? new GenericDatumReader<T>(myCoder.getSchema())
              : new ReflectDatumReader<T>(
                  myCoder.getSchema(), myCoder.getSchema(), myCoder.reflectData.get());
        }
      };

  this.writer =
      new EmptyOnDeserializationThreadLocal<DatumWriter<T>>() {
        private final AvroCoder<T> myCoder = AvroCoder.this;

        @Override
        public DatumWriter<T> initialValue() {
          return myCoder.getType().equals(GenericRecord.class)
              ? new GenericDatumWriter<T>(myCoder.getSchema())
              : new ReflectDatumWriter<T>(myCoder.getSchema(), myCoder.reflectData.get());
        }
      };
}