Java DataFileReader.hasNext方法代码示例

import org.apache.avro.file.DataFileReader; //导入方法依赖的package包/类
/**
 * Reads in binary Avro-encoded entities using the schema stored in the file
 * and prints them out.
 */
public static void readAvroFile(File file) throws IOException {
	GenericDatumReader datum = new GenericDatumReader();
	DataFileReader reader = new DataFileReader(file, datum);

	GenericData.Record record = new GenericData.Record(reader.getSchema());
	while (reader.hasNext()) {
		reader.next(record);
		System.out.println("Name " + record.get("name") + " on "
				+ record.get("Meetup_date") + " attending "
				+ record.get("going") + " organized by  "
				+ record.get("organizer") + " on  " + record.get("topics"));
	}

	reader.close();
}
 

import org.apache.avro.file.DataFileReader; //导入方法依赖的package包/类
@Test
public void testGenericRecord() throws IOException {
	final Path outputPath = new Path(File.createTempFile("avro-output-file", "generic.avro").getAbsolutePath());
	final AvroOutputFormat<GenericRecord> outputFormat = new AvroOutputFormat<>(outputPath, GenericRecord.class);
	Schema schema = new Schema.Parser().parse("{\"type\":\"record\", \"name\":\"user\", \"fields\": [{\"name\":\"user_name\", \"type\":\"string\"}, {\"name\":\"favorite_number\", \"type\":\"int\"}, {\"name\":\"favorite_color\", \"type\":\"string\"}]}");
	outputFormat.setWriteMode(FileSystem.WriteMode.OVERWRITE);
	outputFormat.setSchema(schema);
	output(outputFormat, schema);

	GenericDatumReader<GenericRecord> reader = new GenericDatumReader<>(schema);
	DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(new File(outputPath.getPath()), reader);

	while (dataFileReader.hasNext()) {
		GenericRecord record = dataFileReader.next();
		assertEquals(record.get("user_name").toString(), "testUser");
		assertEquals(record.get("favorite_number"), 1);
		assertEquals(record.get("favorite_color").toString(), "blue");
	}

	//cleanup
	FileSystem fs = FileSystem.getLocalFileSystem();
	fs.delete(outputPath, false);

}
 

import org.apache.avro.file.DataFileReader; //导入方法依赖的package包/类
public void deserUserCompile(){
	// Deserialize Users from disk
	DatumReader<User> userDatumReader = new SpecificDatumReader<User>(User.class);
	DataFileReader<User> dataFileReader = null;
	User user = null;
	try {
		dataFileReader = new DataFileReader<User>(
										new File("/Users/a/Desktop/tmp/users.avro"),
										userDatumReader);
		while (dataFileReader.hasNext()) {
			// Reuse user object by passing it to next(). This saves us from
			// allocating and garbage collecting many objects for files with
			// many items.
			user = dataFileReader.next(user);
			System.out.println(user);
		}
	} catch (IOException e) {
		e.printStackTrace();
	}
}
 

import org.apache.avro.file.DataFileReader; //导入方法依赖的package包/类
/**
 * 动态反序列：通过Schema文件进行动态反序列化操作
 * 
 * @throws IOException
 */
public void MemberInfoDynDeser() throws IOException {
    // 1.schema文件解析
    Parser parser = new Parser();
    Schema mSchema = parser.parse(this.getClass().getResourceAsStream("/Members.avsc"));

    // 2.构建数据读对象
    DatumReader<GenericRecord> mGr = new SpecificDatumReader<GenericRecord>(mSchema);
    DataFileReader<GenericRecord> mDfr = new DataFileReader<GenericRecord>(new File("/Users/a/Desktop/tmp/members.avro"), mGr);
    // 3.从序列化文件中进行数据反序列化取出数据
    GenericRecord gr = null;
    while (mDfr.hasNext()) {
        gr = mDfr.next();
        System.err.println("deser data:" + gr.toString());
    }
    mDfr.close();
    System.out.println("Dyn Builder Ser Start Complete.");
}
 

import org.apache.avro.file.DataFileReader; //导入方法依赖的package包/类
public static void main(String[] args) throws IOException
{
    String filename = args[0] ;
    
    File file=new File(filename) ;
    DatumReader<GenericRecord> reader= new GenericDatumReader<GenericRecord>();
    DataFileReader<GenericRecord> dataFileReader=new DataFileReader<GenericRecord>(file,reader);
    
    while (dataFileReader.hasNext())
    {
        GenericRecord result=dataFileReader.next();
        String output = String.format("%s %s %s %f",
            result.get("sighting_date"), result.get("city"), result.get("shape"), result.get("duration")) ;
        System.out.println(output) ;
    }
}
 

import org.apache.avro.file.DataFileReader; //导入方法依赖的package包/类
public static void main(String[] args) throws IOException
{
    String filename = args[0] ;
    
    File file=new File(filename) ;
    DatumReader<GenericRecord> reader= new GenericDatumReader<GenericRecord>();
    DataFileReader<GenericRecord> dataFileReader=new DataFileReader<GenericRecord>(file,reader);
    
    while (dataFileReader.hasNext())
    {
        GenericRecord result=dataFileReader.next();
        String output = String.format("%s %d",
            result.get("shape"), result.get("count")) ;
        System.out.println(output) ;
    }
}
 

import org.apache.avro.file.DataFileReader; //导入方法依赖的package包/类
/**
 * Reads in binary Avro-encoded entities using a schema that is different
 * from the writer's schema.
 * 
 */
public static void readWithDifferentSchema(File file, Schema newSchema)
		throws IOException {
	GenericDatumReader datum = new GenericDatumReader(newSchema);
	DataFileReader reader = new DataFileReader(file, datum);

	GenericData.Record record = new GenericData.Record(newSchema);
	while (reader.hasNext()) {
		reader.next(record);
		System.out.println("Name " + record.get("name") + " on "
				+ record.get("Meetup_date") + " attending "
				+ record.get("attendance") + " organized by  "
				+ record.get("organizer") 
				+ " at  " + record.get("location"));
	}

	reader.close();
}
 

import org.apache.avro.file.DataFileReader; //导入方法依赖的package包/类
/**
 * Deserialize Users from disk
 */
private void deserializing() {
	long tiempoInicio = System.currentTimeMillis();
	// We create a SpecificDatumReader, analogous to the SpecificDatumWriter we used in serialization, which converts in-memory serialized items into instances of our generated class, in this case User.
	DatumReader<User> userDatumReader = new SpecificDatumReader<User>(User.class);
	// We pass the DatumReader and the previously created File to a DataFileReader, analogous to the DataFileWriter, which reads the data file on disk.
	try {
		File file = createFile();
		@SuppressWarnings("resource")
		DataFileReader<User> dataFileReader = new DataFileReader<User>(file, userDatumReader);
		User user = null;
		// Next we use the DataFileReader to iterate through the serialized Users and print the deserialized object to stdout.
		while (dataFileReader.hasNext()) {
			// Reuse user object by passing it to next(). This saves us from
			// allocating and garbage collecting many objects for files with
			// many items.
			user = dataFileReader.next(user);
			System.out.println(user);
		}
		// DataFileReader no cuenta con un metodo close()
		dataFileReader=null;
	} catch (IOException e) {
		e.printStackTrace();
	}
	terminaProceso("deserializing", tiempoInicio);
}
 

import org.apache.avro.file.DataFileReader; //导入方法依赖的package包/类
/**
 *Reads the avro file
 * @throws IOException
 */
private void readFile() throws IOException {
    // Deserialize Activities from disk
    
    File file = new File(filename);
            
    DatumReader<Activity> activityDatumReader = new SpecificDatumReader<Activity>(Activity.class);
    DataFileReader<Activity> dataFileReader = new DataFileReader<Activity>(file, activityDatumReader);

    Activity activity = null;
    int i = 0;
    
    while (dataFileReader.hasNext() && i < numrecs) {
        i++;
        activity = dataFileReader.next(activity);
        System.out.println(activity);
    }
}
 

import org.apache.avro.file.DataFileReader; //导入方法依赖的package包/类
private static Map<String, Number> computeAggregateFromRawData(File avroFile, Map<String, String> fixedValues) throws Exception {
  long m0Aggregate = 0;
  double m1Aggregate = 0.0;

  DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>();
  DataFileReader<GenericRecord> fileReader = new DataFileReader<GenericRecord>(avroFile, reader);
  GenericRecord record = null;
  while (fileReader.hasNext()) {
    record = fileReader.next(record);
    boolean matches = true;
    for (Map.Entry<String, String> entry : fixedValues.entrySet()) {
      String value = record.get(entry.getKey()).toString();
      if (!value.equals(entry.getValue())) {
        matches = false;
      }
    }

    if (matches) {
      m0Aggregate += (Long) record.get("M0");
      m1Aggregate += (Double) record.get("M1");
    }
  }

  return ImmutableMap.of("M0", m0Aggregate, "M1", m1Aggregate);
}
 

import org.apache.avro.file.DataFileReader; //导入方法依赖的package包/类
public static Object deserializeFromByte(byte[] byteArray)
		throws IOException {
	SeekableByteArrayInput sin = new SeekableByteArrayInput(byteArray);
	System.out.println("length of read input stream " + sin.length());
	DatumReader<?> reader2 = new ReflectDatumReader<>();
	DataFileReader<?> in = new DataFileReader<>(sin, reader2);
	System.out.println(in.getSchema());
	System.out.println(in.hasNext());
	Object returnObject = null;
	System.out.println(in.getSchema().getFullName() );
	while (in.hasNext()) {
		returnObject = in.next();
		
	}
	in.close();
	return returnObject;
}
 

import org.apache.avro.file.DataFileReader; //导入方法依赖的package包/类
/**
 * @param file
 *            where to read from
 * @param schema
 *            Avro records structure
 * @return all values from the file
 * @throws IOException
 */
public static <T extends SpecificRecord> List<T> extractRecords(File file, Schema schema) throws IOException {
    DatumReader<T> reader = new SpecificDatumReader<>(schema);
    DataFileReader<T> fileReader = new DataFileReader<>(file, reader);
    List<T> data = new ArrayList<T>();
    try {
     while (fileReader.hasNext()) {
         data.add(fileReader.next());
     }
    } finally {
    	if(fileReader != null) {
    		fileReader.close();
    	}
    }
    return data;
}
 

import org.apache.avro.file.DataFileReader; //导入方法依赖的package包/类
private DataFileReader<T> getNextNonemptyReader() throws IOException {
	while (fileIterator != null && fileIterator.hasNext()) {
		LocatedFileStatus currentFileStatus = fileIterator.next();
		if (isValidFile(currentFileStatus)) {
			FileSystemPath currPath = new FileSystemPath(
					path.getFileSystem(), currentFileStatus.getPath());
			DataFileReader<T> reader = 
					getSingleFileReader(currPath, readerSchema);
			/** Check if the file contains at least one record */
			if(reader.hasNext()){
				return reader;
			} else {
				reader.close();
			}
		}
	}
	/** fallback */
	return null;
}
 

import org.apache.avro.file.DataFileReader; //导入方法依赖的package包/类
private void loadMembersToPurge(String filename) throws IOException
{
    // TODO: "memberId" column name should be configurable
    DataFileReader<GenericRecord> dataFileReader =
            createDataFileReader(filename, true);
    while (dataFileReader.hasNext())
    {
        GenericRecord record = dataFileReader.next();
        Integer memberId = (Integer) record.get("memberId");
        if (memberId == null)
        {
            throw new NullPointerException("memberId is null");
        }
        membersToPurge.add(((Number) record.get("memberId")).intValue());
    }
    dataFileReader.close();
}
 

import org.apache.avro.file.DataFileReader; //导入方法依赖的package包/类
static <T> T runOnPreview(byte[] bits, AvroPreviewProcessor<T> processor) throws IOException {
  DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>();
  SeekableByteArrayInput sbai = new SeekableByteArrayInput(bits);
  DataFileReader<GenericRecord> dataFileReader = null;

  try {
    dataFileReader = new DataFileReader<>(sbai, datumReader);
    int headerLen = (int) dataFileReader.previousSync();
    byte[] header = Arrays.copyOf(bits, headerLen);
    if (dataFileReader.hasNext()) {
      GenericRecord gr = dataFileReader.next();
      return processor.process(header, gr, dataFileReader.getBlockCount(), dataFileReader.getBlockSize());
    } else {
      throw new RuntimeException("Empty Avro file - cannot run preview! ");
    }
  } finally {
    try { if (dataFileReader!=null) dataFileReader.close(); } catch (IOException safeToIgnore) {}
  }
}