Java DataFileWriter.create方法代码示例

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
/**
 * Writes the given {@link DataEvent}s from the eventStream into the
 * {@link OutputStream} using avro's object container format (see
 * {@link DataFileWriter}). Please note: As this method is creating the
 * {@link OutputStream} via the {@link Supplier}, the {@link OutputStream}
 * is as well closed by this method.
 * 
 * 
 * @param eventStream
 *            a stream of valid and normalized {@link DataEvent}.
 * @param out
 *            a {@link Supplier} of an output stream
 * @throws IOException
 */
public static void writeToStream(Stream<DataEvent> eventStream, Supplier<OutputStream> outSupplier) throws IOException {
	final OutputStream out = outSupplier.get();
	@Cleanup
	final DataFileWriter<DataEvent> writer = new DataFileWriter<>(DataEventSerializer.getWRITER());
	writer.setSyncInterval(1024 * 1024);
	writer.setCodec(CodecFactory.deflateCodec(9));
	writer.setMeta("created_at", new Date().getTime());
	writer.create(DataEvent.SCHEMA$, out);

	eventStream.forEach(event -> {
		try {
			writer.append(event);
		} catch (Exception e) {
			throw new RuntimeException(e);
		}
	});
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
/**
 * 动态序列化：通过动态解析Schema文件进行内容设置，并序列化内容
 * 
 * @throws IOException
 */
public void MemberInfoDynSer() throws IOException {
    // 1.解析schema文件内容
    Parser parser = new Parser();
    Schema mSchema = parser.parse(this.getClass().getResourceAsStream("/Members.avsc"));
    // 2.构建数据写对象
    DatumWriter<GenericRecord> mGr = new SpecificDatumWriter<GenericRecord>(mSchema);
    DataFileWriter<GenericRecord> mDfw = new DataFileWriter<GenericRecord>(mGr);
    // 3.创建序列化文件
    mDfw.create(mSchema, new File("/Users/a/Desktop/tmp/members.avro"));
    // 4.添加序列化数据
    for (int i = 0; i < 20; i++) {
        GenericRecord gr = new GenericData.Record(mSchema);
        int r = i * new Random().nextInt(50);
        gr.put("userName", "light-" + r);
        gr.put("userPwd", "2016-" + r);
        gr.put("realName", "滔滔" + r + "号");
        mDfw.append(gr);
    }
    // 5.关闭数据文件写对象
    mDfw.close();
    System.out.println("Dyn Builder Ser Start Complete.");
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
@Override
public void close(TaskAttemptContext context) throws IOException {
    // Create an Avro container file and a writer to it.
    DataFileWriter<K> avroFileWriter;
    avroFileWriter = new DataFileWriter<K>(new ReflectDatumWriter<K>(writerSchema));
    avroFileWriter.setCodec(compressionCodec);

    // Writes the meta-data.
    avroFileWriter.setMeta(Constants.AVRO_NUMBER_OF_RECORDS, this.numberOfRecords);

    // Writes the file.
    avroFileWriter.create(this.writerSchema, this.outputStream);
    for (AvroKey<K> record : this.recordsList)
        avroFileWriter.append(record.datum());

    // Close the stream.
    avroFileWriter.close();
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
public static byte[] putRecords(Collection<SinkRecord> records, AvroData avroData) throws IOException {
  final DataFileWriter<Object> writer = new DataFileWriter<>(new GenericDatumWriter<>());
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  Schema schema = null;
  for (SinkRecord record : records) {
    if (schema == null) {
      schema = record.valueSchema();
      org.apache.avro.Schema avroSchema = avroData.fromConnectSchema(schema);
      writer.create(avroSchema, out);
    }
    Object value = avroData.fromConnectData(schema, record.value());
    // AvroData wraps primitive types so their schema can be included. We need to unwrap
    // NonRecordContainers to just their value to properly handle these types
    if (value instanceof NonRecordContainer) {
      value = ((NonRecordContainer) value).getValue();
    }
    writer.append(value);
  }
  writer.flush();
  return out.toByteArray();
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
public static void writeToAvro(File srcPath,
        OutputStream outputStream)
        throws IOException {
  DataFileWriter<Object> writer =
          new DataFileWriter<Object>(
              new GenericDatumWriter<Object>())
              .setSyncInterval(100);                 //<co id="ch02_smallfilewrite_comment2"/>
  writer.setCodec(CodecFactory.snappyCodec());   //<co id="ch02_smallfilewrite_comment3"/>
  writer.create(SCHEMA, outputStream);           //<co id="ch02_smallfilewrite_comment4"/>
  for (Object obj : FileUtils.listFiles(srcPath, null, false)) {
    File file = (File) obj;
    String filename = file.getAbsolutePath();
    byte content[] = FileUtils.readFileToByteArray(file);
    GenericRecord record = new GenericData.Record(SCHEMA);  //<co id="ch02_smallfilewrite_comment5"/>
    record.put(FIELD_FILENAME, filename);                   //<co id="ch02_smallfilewrite_comment6"/>
    record.put(FIELD_CONTENTS, ByteBuffer.wrap(content));   //<co id="ch02_smallfilewrite_comment7"/>
    writer.append(record);                                  //<co id="ch02_smallfilewrite_comment8"/>
    System.out.println(
            file.getAbsolutePath()
            + ": "
            + DigestUtils.md5Hex(content));
  }

  IOUtils.cleanup(null, writer);
  IOUtils.cleanup(null, outputStream);
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
/**
 * Serialize our Users to disk.
 */
private void serializing(List<User> listUsers) {
	long tiempoInicio = System.currentTimeMillis();
	// We create a DatumWriter, which converts Java objects into an in-memory serialized format.
	// The SpecificDatumWriter class is used with generated classes and extracts the schema from the specified generated type.
	DatumWriter<User> userDatumWriter = new SpecificDatumWriter<User>(User.class);
	// We create a DataFileWriter, which writes the serialized records, as well as the schema, to the file specified in the dataFileWriter.create call.
	DataFileWriter<User> dataFileWriter = new DataFileWriter<User>(userDatumWriter);

	try {
		File file = createFile();
		dataFileWriter.create(((User) listUsers.get(0)).getSchema(), file);
		for (User user : listUsers) {
			// We write our users to the file via calls to the dataFileWriter.append method.
			dataFileWriter.append(user);
		}
		// When we are done writing, we close the data file.
		dataFileWriter.close();
	} catch (IOException e) {
		e.printStackTrace();
	}
	terminaProceso("serializing", tiempoInicio);
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
@Override
public byte[] serialise(final Object object) throws SerialisationException {
    Schema schema = ReflectData.get().getSchema(object.getClass());
    DatumWriter<Object> datumWriter = new ReflectDatumWriter<>(schema);
    DataFileWriter<Object> dataFileWriter = new DataFileWriter<>(datumWriter);
    ByteArrayOutputStream byteOut = new ByteArrayOutputStream();
    try {
        dataFileWriter.create(schema, byteOut);
        dataFileWriter.append(object);
        dataFileWriter.flush();
    } catch (final IOException e) {
        throw new SerialisationException("Unable to serialise given object of class: " + object.getClass().getName(), e);
    } finally {
        close(dataFileWriter);
    }
    return byteOut.toByteArray();
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
public static void main(String[] args) throws IOException {
  // Open data file
  File file = new File(PATH);
  if (file.getParentFile() != null) {
    file.getParentFile().mkdirs();
  }
  DatumWriter<User> userDatumWriter = new SpecificDatumWriter<User>(User.class);
  DataFileWriter<User> dataFileWriter = new DataFileWriter<User>(userDatumWriter);
  dataFileWriter.create(User.SCHEMA$, file);

  // Create random users
  User user;
  Random random = new Random();
  for (int i = 0; i < USERS; i++) {
    user = new User("user", null, COLORS[random.nextInt(COLORS.length)]);
    dataFileWriter.append(user);
    System.out.println(user);
  }

  dataFileWriter.close();
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
public static void createFileIfNotExists(BlockSchema fileSchema, String path) throws IOException
{
    Configuration conf = new JobConf();
    FileSystem fs = FileSystem.get(conf);
    if (fs.exists(new Path(path)))
        return;

    Schema avroSchema = convertFromBlockSchema("CUBERT_MV_RECORD", fileSchema);
    System.out.println("Creating avro file with schema = " + avroSchema);
    GenericDatumWriter<GenericRecord> datumWriter =
            new GenericDatumWriter<GenericRecord>(avroSchema);
    DataFileWriter<GenericRecord> writer =
            new DataFileWriter<GenericRecord>(datumWriter);

    FSDataOutputStream fout =
            FileSystem.create(fs,
                              new Path(path),
                              new FsPermission(FsAction.ALL,
                                               FsAction.READ_EXECUTE,
                                               FsAction.READ_EXECUTE));
    writer.create(avroSchema, fout);
    writer.flush();
    writer.close();

}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
@Override
public RecordWriter<NullWritable, Object> getRecordWriter(TaskAttemptContext context) throws IOException, InterruptedException {

    if (schema == null)
        throw new IOException("Must provide a schema");
    
    Configuration conf = context.getConfiguration();

    DataFileWriter<Object> writer = new DataFileWriter<Object>(new PigAvroDatumWriter(schema));

    if (FileOutputFormat.getCompressOutput(context)) {
        int level = conf.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL);
        String codecName = conf.get(OUTPUT_CODEC, DEFLATE_CODEC);
        CodecFactory factory = codecName.equals(DEFLATE_CODEC)
            ? CodecFactory.deflateCodec(level)
            : CodecFactory.fromString(codecName);
        writer.setCodec(factory);
    }

    Path path = getDefaultWorkFile(context, EXT);
    writer.create(schema, path.getFileSystem(conf).create(path));
    return new PigAvroRecordWriter(writer);
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
public void create(String dirname, String filename, GenericData.Record[] records) throws Exception {


        Schema schema = records[0].getSchema();

        File dir = new File(tmpFolder.getRoot(), dirname);

        if (!dir.exists()) {
            dir.mkdirs();
        }

        File file = new File(dir, filename);
        DatumWriter<GenericRecord> datumWriter = new GenericDatumWriter<GenericRecord>(schema) {
            @Override
            protected void writeBytes(Object datum, Encoder out) throws IOException {
                super.writeBytes(ByteBuffer.wrap((byte[]) datum), out);
            }
        };
        DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<GenericRecord>(datumWriter);
        dataFileWriter.create(schema, file);
        TestSchema.append(dataFileWriter, records);
        dataFileWriter.close();

    }
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
@SuppressWarnings("resource")
 public static void writeToAvro(File inputFile, OutputStream outputStream)
     throws IOException {

DataFileWriter<Stock> writer = //a writer to write Avro
       new DataFileWriter<Stock>(new SpecificDatumWriter<Stock>()).setSyncInterval(100);      

   writer.setCodec(CodecFactory.snappyCodec());   
   writer.create(Stock.SCHEMA$, outputStream);    //identify the schema

   for(String line: FileUtils.readLines(inputFile)) {
     writer.append(createStock(line));     //write to Avro file
   }

   IOUtils.closeStream(writer);
   IOUtils.closeStream(outputStream);
 }
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
@Override
public RecordWriter<AvroWrapper<T>, NullWritable> getRecordWriter(
  TaskAttemptContext context) throws IOException, InterruptedException {

  boolean isMapOnly = context.getNumReduceTasks() == 0;
  Schema schema =
    isMapOnly ? AvroJob.getMapOutputSchema(context.getConfiguration())
      : AvroJob.getOutputSchema(context.getConfiguration());

  final DataFileWriter<T> WRITER =
    new DataFileWriter<T>(new ReflectDatumWriter<T>());

  configureDataFileWriter(WRITER, context);

  Path path = getDefaultWorkFile(context, EXT);
  WRITER.create(schema,
    path.getFileSystem(context.getConfiguration()).create(path));

  return new RecordWriter<AvroWrapper<T>, NullWritable>() {
    @Override
    public void write(AvroWrapper<T> wrapper, NullWritable ignore)
      throws IOException {
      WRITER.append(wrapper.datum());
    }

    @Override
    public void close(TaskAttemptContext taskAttemptContext)
      throws IOException, InterruptedException {
      WRITER.close();
    }
  };
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
/**
 * Create a data file that gets exported to the db.
 * @param fileNum the number of the file (for multi-file export)
 * @param numRecords how many records to write to the file.
 */
protected void createAvroFile(int fileNum, int numRecords,
    ColumnGenerator... extraCols) throws IOException {

  Path tablePath = getTablePath();
  Path filePath = new Path(tablePath, "part" + fileNum);

  Configuration conf = new Configuration();
  if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
    conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
  }
  FileSystem fs = FileSystem.get(conf);
  fs.mkdirs(tablePath);
  OutputStream os = fs.create(filePath);

  Schema schema = buildAvroSchema(extraCols);
  DatumWriter<GenericRecord> datumWriter =
    new GenericDatumWriter<GenericRecord>();
  DataFileWriter<GenericRecord> dataFileWriter =
    new DataFileWriter<GenericRecord>(datumWriter);
  dataFileWriter.create(schema, os);

  for (int i = 0; i < numRecords; i++) {
    GenericRecord record = new GenericData.Record(schema);
    record.put("id", i);
    record.put("msg", getMsgPrefix() + i);
    addExtraColumns(record, i, extraCols);
    dataFileWriter.append(record);
  }

  dataFileWriter.close();
  os.close();
}
 

import org.apache.avro.file.DataFileWriter; //导入方法依赖的package包/类
public void addUserCompile(){
	User user1 = new User();
	user1.setName("王light");
	user1.setFavoriteNumber(66);
	user1.setFavoriteColor("浅蓝色");
	
	// Alternate constructor
	User user2 = new User("魏Sunny", 88, "red");
	
	// Construct via builder
	User user3 = User.newBuilder()
	             .setName("王Sam")
	             .setFavoriteColor("blue")
	             .setFavoriteNumber(2011)
	             .build();
	
	DatumWriter<User> userDatumWriter = new SpecificDatumWriter<User>(User.class);
	DataFileWriter<User> dataFileWriter = new DataFileWriter<User>(userDatumWriter);
	try {
		dataFileWriter.create(user1.getSchema(), new File("/Users/a/Desktop/tmp/users.avro"));
		dataFileWriter.append(user1);
		dataFileWriter.append(user2);
		dataFileWriter.append(user3);
		dataFileWriter.close();
	} catch (IOException e) {
		e.printStackTrace();
	}
}