Java DataFileWriter.append方法代码示例

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
/**
 * 动态序列化：通过动态解析Schema文件进行内容设置，并序列化内容
 * 
 * @throws IOException
 */
public void MemberInfoDynSer() throws IOException {
    // 1.解析schema文件内容
    Parser parser = new Parser();
    Schema mSchema = parser.parse(this.getClass().getResourceAsStream("/Members.avsc"));
    // 2.构建数据写对象
    DatumWriter<GenericRecord> mGr = new SpecificDatumWriter<GenericRecord>(mSchema);
    DataFileWriter<GenericRecord> mDfw = new DataFileWriter<GenericRecord>(mGr);
    // 3.创建序列化文件
    mDfw.create(mSchema, new File("/Users/a/Desktop/tmp/members.avro"));
    // 4.添加序列化数据
    for (int i = 0; i < 20; i++) {
        GenericRecord gr = new GenericData.Record(mSchema);
        int r = i * new Random().nextInt(50);
        gr.put("userName", "light-" + r);
        gr.put("userPwd", "2016-" + r);
        gr.put("realName", "滔滔" + r + "号");
        mDfw.append(gr);
    }
    // 5.关闭数据文件写对象
    mDfw.close();
    System.out.println("Dyn Builder Ser Start Complete.");
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
@Override
public void close(TaskAttemptContext context) throws IOException {
    // Create an Avro container file and a writer to it.
    DataFileWriter<K> avroFileWriter;
    avroFileWriter = new DataFileWriter<K>(new ReflectDatumWriter<K>(writerSchema));
    avroFileWriter.setCodec(compressionCodec);

    // Writes the meta-data.
    avroFileWriter.setMeta(Constants.AVRO_NUMBER_OF_RECORDS, this.numberOfRecords);

    // Writes the file.
    avroFileWriter.create(this.writerSchema, this.outputStream);
    for (AvroKey<K> record : this.recordsList)
        avroFileWriter.append(record.datum());

    // Close the stream.
    avroFileWriter.close();
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
public static byte[] putRecords(Collection<SinkRecord> records, AvroData avroData) throws IOException {
  final DataFileWriter<Object> writer = new DataFileWriter<>(new GenericDatumWriter<>());
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  Schema schema = null;
  for (SinkRecord record : records) {
    if (schema == null) {
      schema = record.valueSchema();
      org.apache.avro.Schema avroSchema = avroData.fromConnectSchema(schema);
      writer.create(avroSchema, out);
    }
    Object value = avroData.fromConnectData(schema, record.value());
    // AvroData wraps primitive types so their schema can be included. We need to unwrap
    // NonRecordContainers to just their value to properly handle these types
    if (value instanceof NonRecordContainer) {
      value = ((NonRecordContainer) value).getValue();
    }
    writer.append(value);
  }
  writer.flush();
  return out.toByteArray();
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
public void generateAvroFile(Schema schema, File file, long recourdCount) throws IOException {
  DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema);
  DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<>(writer);
  dataFileWriter.create(schema, file);

  for(long i = 0; i < recourdCount; i++) {
    GenericRecord datum = new GenericData.Record(schema);
    datum.put("b", i % 2 == 0);
    datum.put("s", String.valueOf(i));
    datum.put("l", i);
    datum.put("l100", i % 100);
    datum.put("s100", String.valueOf(i%100));
    dataFileWriter.append(datum);
  }

  dataFileWriter.close();
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
public static void writeToAvro(File inputFile, OutputStream outputStream)
    throws IOException {

  DataFileWriter<GenericRecord> writer =
      new DataFileWriter<GenericRecord>(
          new GenericDatumWriter<GenericRecord>());

  writer.setCodec(CodecFactory.snappyCodec());
  writer.create(SCHEMA, outputStream);

  for (Stock stock : AvroStockUtils.fromCsvFile(inputFile)) {

    AvroKeyValue<CharSequence, Stock> record
        = new AvroKeyValue<CharSequence, Stock>(new GenericData.Record(SCHEMA));
    record.setKey(stock.getSymbol());
    record.setValue(stock);

    writer.append(record.get());
  }

  IOUtils.closeStream(writer);
  IOUtils.closeStream(outputStream);
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
/**
 * Read JSON file divided into lines, where each one corresponds to a
 * record. Next, save the extracted records in a data store.
 */
public static void convertToDataStore(Schema inputSchema,
		InputStream input, FileSystemPath outputPath) throws IOException {
	JsonStreamReader<GenericRecord> reader = new JsonStreamReader<GenericRecord>(
			inputSchema, input, GenericRecord.class);
	DataFileWriter<GenericRecord> writer = DataStore.create(outputPath,
			inputSchema);
	try {
		while (reader.hasNext()) {
			Object obj = reader.next();
			GenericRecord record = (GenericRecord) obj;
			writer.append(record);
		}
	} finally {
		if (writer != null) {
			writer.close();
		}
		if (reader != null) {
			reader.close();
		}
	}
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
/**
 * Serialize our Users to disk.
 */
private void serializing(List<User> listUsers) {
	long tiempoInicio = System.currentTimeMillis();
	// We create a DatumWriter, which converts Java objects into an in-memory serialized format.
	// The SpecificDatumWriter class is used with generated classes and extracts the schema from the specified generated type.
	DatumWriter<User> userDatumWriter = new SpecificDatumWriter<User>(User.class);
	// We create a DataFileWriter, which writes the serialized records, as well as the schema, to the file specified in the dataFileWriter.create call.
	DataFileWriter<User> dataFileWriter = new DataFileWriter<User>(userDatumWriter);

	try {
		File file = createFile();
		dataFileWriter.create(((User) listUsers.get(0)).getSchema(), file);
		for (User user : listUsers) {
			// We write our users to the file via calls to the dataFileWriter.append method.
			dataFileWriter.append(user);
		}
		// When we are done writing, we close the data file.
		dataFileWriter.close();
	} catch (IOException e) {
		e.printStackTrace();
	}
	terminaProceso("serializing", tiempoInicio);
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
private byte[] avroTestData(List<Schema.Field> fields, List<Map<String, Object>> records)
        throws IOException
{
    Schema schema = Schema.createRecord("testdata", null, null, false);
    schema.setFields(fields);

    ByteArrayOutputStream out = new ByteArrayOutputStream();
    GenericDatumWriter<GenericData.Record> datum = new GenericDatumWriter<>(schema);
    DataFileWriter<GenericData.Record> writer = new DataFileWriter<>(datum);
    writer.create(schema, out);
    for (Map<String, Object> record : records) {
        GenericData.Record r = new GenericData.Record(schema);
        for (Map.Entry<String, Object> item : record.entrySet()) {
            r.put(item.getKey(), item.getValue());
        }
        writer.append(r);
    }
    writer.close();

    return out.toByteArray();
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
public static void main(String[] args) throws IOException {
  // Open data file
  File file = new File(PATH);
  if (file.getParentFile() != null) {
    file.getParentFile().mkdirs();
  }
  DatumWriter<User> userDatumWriter = new SpecificDatumWriter<User>(User.class);
  DataFileWriter<User> dataFileWriter = new DataFileWriter<User>(userDatumWriter);
  dataFileWriter.create(User.SCHEMA$, file);

  // Create random users
  User user;
  Random random = new Random();
  for (int i = 0; i < USERS; i++) {
    user = new User("user", null, COLORS[random.nextInt(COLORS.length)]);
    dataFileWriter.append(user);
    System.out.println(user);
  }

  dataFileWriter.close();
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
@Override
protected void processFile(
		final URL file,
		final String typeName,
		final AvroFormatPlugin<?, ?> plugin,
		final StageRunData runData ) {
	final DataFileWriter writer = runData.getWriter(
			typeName,
			plugin);
	if (writer != null) {
		final Object[] objs = plugin.toAvroObjects(file);
		for (final Object obj : objs) {
			try {
				writer.append(obj);
			}
			catch (final IOException e) {
				LOGGER.error(
						"Cannot append data to sequence file",
						e);
			}
		}
	}
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
private byte[] encodeSpecificRecord(Object data) {
  SpecificRecord record = (SpecificRecord) data;

  ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();

  DatumWriter userDatumWriter = new SpecificDatumWriter(data.getClass());
  DataFileWriter dataFileWriter = new DataFileWriter(userDatumWriter);
  try {
    dataFileWriter.create(record.getSchema(), byteArrayOutputStream);
    dataFileWriter.append(data);
    dataFileWriter.close();

  } catch (IOException e) {
    e.printStackTrace();
  }
  return byteArrayOutputStream.toByteArray();
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
@SuppressWarnings("resource")
 public static void writeToAvro(File inputFile, OutputStream outputStream)
     throws IOException {

DataFileWriter<Stock> writer = //a writer to write Avro
       new DataFileWriter<Stock>(new SpecificDatumWriter<Stock>()).setSyncInterval(100);      

   writer.setCodec(CodecFactory.snappyCodec());   
   writer.create(Stock.SCHEMA$, outputStream);    //identify the schema

   for(String line: FileUtils.readLines(inputFile)) {
     writer.append(createStock(line));     //write to Avro file
   }

   IOUtils.closeStream(writer);
   IOUtils.closeStream(outputStream);
 }
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
@Override
public RecordWriter<AvroWrapper<T>, NullWritable> getRecordWriter(
  TaskAttemptContext context) throws IOException, InterruptedException {

  boolean isMapOnly = context.getNumReduceTasks() == 0;
  Schema schema =
    isMapOnly ? AvroJob.getMapOutputSchema(context.getConfiguration())
      : AvroJob.getOutputSchema(context.getConfiguration());

  final DataFileWriter<T> WRITER =
    new DataFileWriter<T>(new ReflectDatumWriter<T>());

  configureDataFileWriter(WRITER, context);

  Path path = getDefaultWorkFile(context, EXT);
  WRITER.create(schema,
    path.getFileSystem(context.getConfiguration()).create(path));

  return new RecordWriter<AvroWrapper<T>, NullWritable>() {
    @Override
    public void write(AvroWrapper<T> wrapper, NullWritable ignore)
      throws IOException {
      WRITER.append(wrapper.datum());
    }

    @Override
    public void close(TaskAttemptContext taskAttemptContext)
      throws IOException, InterruptedException {
      WRITER.close();
    }
  };
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
/**
 * Create a data file that gets exported to the db.
 * @param fileNum the number of the file (for multi-file export)
 * @param numRecords how many records to write to the file.
 */
protected void createAvroFile(int fileNum, int numRecords,
    ColumnGenerator... extraCols) throws IOException {

  Path tablePath = getTablePath();
  Path filePath = new Path(tablePath, "part" + fileNum);

  Configuration conf = new Configuration();
  if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
    conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
  }
  FileSystem fs = FileSystem.get(conf);
  fs.mkdirs(tablePath);
  OutputStream os = fs.create(filePath);

  Schema schema = buildAvroSchema(extraCols);
  DatumWriter<GenericRecord> datumWriter =
    new GenericDatumWriter<GenericRecord>();
  DataFileWriter<GenericRecord> dataFileWriter =
    new DataFileWriter<GenericRecord>(datumWriter);
  dataFileWriter.create(schema, os);

  for (int i = 0; i < numRecords; i++) {
    GenericRecord record = new GenericData.Record(schema);
    record.put("id", i);
    record.put("msg", getMsgPrefix() + i);
    addExtraColumns(record, i, extraCols);
    dataFileWriter.append(record);
  }

  dataFileWriter.close();
  os.close();
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
public void addUserCompile(){
	User user1 = new User();
	user1.setName("王light");
	user1.setFavoriteNumber(66);
	user1.setFavoriteColor("浅蓝色");
	
	// Alternate constructor
	User user2 = new User("魏Sunny", 88, "red");
	
	// Construct via builder
	User user3 = User.newBuilder()
	             .setName("王Sam")
	             .setFavoriteColor("blue")
	             .setFavoriteNumber(2011)
	             .build();
	
	DatumWriter<User> userDatumWriter = new SpecificDatumWriter<User>(User.class);
	DataFileWriter<User> dataFileWriter = new DataFileWriter<User>(userDatumWriter);
	try {
		dataFileWriter.create(user1.getSchema(), new File("/Users/a/Desktop/tmp/users.avro"));
		dataFileWriter.append(user1);
		dataFileWriter.append(user2);
		dataFileWriter.append(user3);
		dataFileWriter.close();
	} catch (IOException e) {
		e.printStackTrace();
	}
}