Java GenericRecordBuilder类代码示例

import org.apache.avro.generic.GenericRecordBuilder; //导入依赖的package包/类
@Test
public void testMaps() throws Exception {
  Schema schema = Maps.SCHEMA$;

  GenericRecordBuilder builder = new GenericRecordBuilder(schema);
  builder.set("maps", ImmutableMap.of("key1", ImmutableMap.of("value1", 1, "value2", 2), "key2",
      ImmutableMap.of(), "key3", ImmutableMap.of("value3", 3)));
  Record record1 = builder.build();

  String json = "{\"maps\": {\"key1\": {\"value1\": 1, \"value2\": 2}, \"key2\": {}, \"key3\": {\"value3\": 3}}}";
  DBObject object = (DBObject) JSON.parse(json);
  Record record2 = RecordConverter.toRecord(schema, object, getClass().getClassLoader());

  // Convert into JsonNode before comparison, so the maps equal even if keys are reordered.
  assertThat(JSON.parse(AvroHelper.toSimpleJson(schema, record2)), is(JSON.parse(AvroHelper.toSimpleJson(schema, record1))));
}
 

import org.apache.avro.generic.GenericRecordBuilder; //导入依赖的package包/类
@Test
public void publish_shouldThrowAnEncodeTransportExceptionForAnInvalidTransportMessage() throws Exception {

	// expect
	exception.expect(EncodeMessageContentException.class);
	exception.expectCause(IsInstanceOf.<Throwable>instanceOf(ClassCastException.class));
	exception.expectMessage("Failed to publish message: Failed to encode message content");

	// given
	Context context = mock(Context.class);
	when(context.getSchema())
			.thenReturn(new Schema.Parser().parse("{\"name\":\"test\",\"type\":\"record\",\"fields\":[]}"));
	when(context.getDataBuffer()).thenReturn(ByteBuffer.wrap("{}".getBytes()));

	GenericRecordBuilder builder = new GenericRecordBuilder(schema);
	builder.set("testString", Boolean.TRUE);
	Record record = builder.build();

	// when
	publisher.publish(context, record);

}
 

import org.apache.avro.generic.GenericRecordBuilder; //导入依赖的package包/类
@Test
public void publish_shouldThrowAnEncodeTransportExceptionForAnInvalidTransportContext() throws Exception {

	// expect
	exception.expect(EncodeTransportException.class);
	exception.expectCause(IsInstanceOf.<Throwable>instanceOf(NullPointerException.class));
	exception.expectMessage("Failed to publish message: Failed to encode transport");

	// given
	Context context = mock(Context.class);
	when(context.getSchema())
			.thenReturn(new Schema.Parser().parse("{\"name\":\"test\",\"type\":\"record\",\"fields\":[]}"));
	when(context.getDataBuffer()).thenReturn(null);

	GenericRecordBuilder builder = new GenericRecordBuilder(schema);
	builder.set("testString", "testString");
	Record record = builder.build();

	// when
	publisher.publish(context, record);

}
 

import org.apache.avro.generic.GenericRecordBuilder; //导入依赖的package包/类
@Test
public void publish_shouldPublishTheMessage() throws Exception {

	// given
	Context context = mock(Context.class);
	when(context.getSchema())
			.thenReturn(new Schema.Parser().parse("{\"name\":\"test\",\"type\":\"record\",\"fields\":[]}"));
	when(context.getDataBuffer()).thenReturn(ByteBuffer.wrap("{}".getBytes()));

	GenericRecordBuilder builder = new GenericRecordBuilder(schema);
	builder.set("testString", "test");
	Record record = builder.build();

	// when
	publisher.publish(context, record);

}
 

import org.apache.avro.generic.GenericRecordBuilder; //导入依赖的package包/类
@Test
public void testIncompatibleSchemas() throws EventDeliveryException {
  final DatasetSink sink = sink(in, config);

  GenericRecordBuilder builder = new GenericRecordBuilder(
      INCOMPATIBLE_SCHEMA);
  GenericData.Record rec = builder.set("username", "koala").build();
  putToChannel(in, event(rec, INCOMPATIBLE_SCHEMA, null, false));

  // run the sink
  sink.start();
  assertThrows("Should fail", EventDeliveryException.class,
      new Callable() {
        @Override
        public Object call() throws EventDeliveryException {
          sink.process();
          return null;
        }
      });
  sink.stop();

  Assert.assertEquals("Should have rolled back",
      expected.size() + 1, remaining(in));
}
 

import org.apache.avro.generic.GenericRecordBuilder; //导入依赖的package包/类
private void writeRowsHelper(List<TableRow> rows, Schema avroSchema,
                             String destinationPattern, int shard) throws IOException {
  String filename = destinationPattern.replace("*", String.format("%012d", shard));
  try (WritableByteChannel channel = FileSystems.create(
      FileSystems.matchNewResource(filename, false /* isDirectory */), MimeTypes.BINARY);
       DataFileWriter<GenericRecord> tableRowWriter =
           new DataFileWriter<>(new GenericDatumWriter<GenericRecord>(avroSchema))
               .create(avroSchema, Channels.newOutputStream(channel))) {
    for (Map<String, Object> record : rows) {
      GenericRecordBuilder genericRecordBuilder = new GenericRecordBuilder(avroSchema);
      for (Map.Entry<String, Object> field : record.entrySet()) {
        genericRecordBuilder.set(field.getKey(), field.getValue());
      }
      tableRowWriter.append(genericRecordBuilder.build());
    }
  } catch (IOException e) {
    throw new IllegalStateException(
        String.format("Could not create destination for extract job %s", filename), e);
  }
}
 

import org.apache.avro.generic.GenericRecordBuilder; //导入依赖的package包/类
/**
 * Parse the entity from the body in JSON of the given event.
 * 
 * @param event
 *            The event to parse.
 * @param reuse
 *            If non-null, this may be reused and returned from this method.
 * @return The parsed entity as a GenericRecord.
 * @throws EventDeliveryException
 *             A recoverable error such as an error downloading the schema
 *             from the URL has occurred.
 * @throws NonRecoverableEventException
 *             A non-recoverable error such as an unparsable schema or
 *             entity has occurred.
 */
@Override
public GenericRecord parse(Event event, GenericRecord reuse)
		throws EventDeliveryException, NonRecoverableEventException {
	
	JsonObject parser = new JsonParser().parse(new String(event.getBody())).getAsJsonObject();
	
	GenericRecordBuilder recordBuilder = new GenericRecordBuilder(datasetSchema);
	for (Field field:datasetSchema.getFields()) {
		String at_header = field.getProp(FIELD_AT_HEADER_PROPERTY);
		
		if(at_header != null && at_header.equals(Boolean.TRUE.toString())){
			recordBuilder.set(field.name(), event.getHeaders().get(field.name()));
		}else{
			JsonElement element = parser.get(field.name());
			
			recordBuilder.set(field.name(), getElementAsType(field.schema(), element));
		}
	}

	return recordBuilder.build();
}
 

import org.apache.avro.generic.GenericRecordBuilder; //导入依赖的package包/类
@Test
public void testDefaultValues() throws Exception {
  Schema schema = Defaults.SCHEMA$;

  GenericRecordBuilder builder = new GenericRecordBuilder(schema).set("id", "1234");
  Record record1 = builder.build();

  String json = "{\"id\": \"1234\"}";
  BSONObject object = (BSONObject) JSON.parse(json);
  Record record2 = RecordConverter.toRecord(schema, object, getClass().getClassLoader());

  assertThat(record2, is(record1));
  assertThat(AvroHelper.toSimpleJson(schema, record2), is(AvroHelper.toSimpleJson(schema, record1)));

  assertEquals(record2.get("id"), "1234");
  assertNull(record2.get("s"));
  assertTrue((Boolean) record2.get("b"));
  assertEquals(((Record) record2.get("r")).get("f"), "value");
  assertEquals(((Record) record2.get("r")).get("l"), 1234l);
}
 

import org.apache.avro.generic.GenericRecordBuilder; //导入依赖的package包/类
@Test
public void testArrays() throws Exception {
  Schema schema = Arrays.SCHEMA$;

  GenericRecordBuilder builder = new GenericRecordBuilder(schema);
  builder.set("arrays", ImmutableList.of(ImmutableList.of(ImmutableList.of(1, 2, 3),
      ImmutableList.of()), ImmutableList.of(ImmutableList.of(4), ImmutableList.of()),
      ImmutableList.of(ImmutableList.of())));
  Record record1 = builder.build();

  String json = "{\"arrays\": [[[1, 2, 3], []], [[4], []], [[]]]}";
  BSONObject object = (BSONObject) JSON.parse(json);
  Record record2 = RecordConverter.toRecord(schema, object, getClass().getClassLoader());

  assertThat(record2, is(record1));
  assertThat(AvroHelper.toSimpleJson(schema, record2), is(AvroHelper.toSimpleJson(schema, record1)));
}
 

import org.apache.avro.generic.GenericRecordBuilder; //导入依赖的package包/类
@Test
public void testCopyFieldsValues() {
    Schema intSchema = SchemaBuilder.record("intSchema")
            .fields()
            .name("a").type().intType().noDefault()
            .endRecord();
    GenericRecord intRecord = new GenericRecordBuilder(intSchema)
            .set("a", 1)
            .build();

    Schema stringSchema = SchemaBuilder.record("intSchema")
            .fields()
            .name("a").type().stringType().noDefault()
            .endRecord();
    GenericRecordBuilder stringRecordBuilder = new GenericRecordBuilder(stringSchema)
            .set("a", "s");
    TypeConverterUtils.copyFieldsValues(intRecord, stringRecordBuilder);
    GenericRecord stringRecord = stringRecordBuilder.build();
    Assert.assertEquals(intRecord.get("a"), stringRecord.get("a"));
}
 

import org.apache.avro.generic.GenericRecordBuilder; //导入依赖的package包/类
static File generateAvroPrimitiveTypes(File parentDir, String filename, int nrows, Date date) throws IOException {
  File f = new File(parentDir, filename);
  Schema schema = new Schema.Parser().parse(Resources.getResource("PrimitiveAvro.avsc").openStream());
  AvroParquetWriter<GenericRecord> writer = new AvroParquetWriter<GenericRecord>(new Path(f.getPath()), schema);
  try {
    DateFormat format = new SimpleDateFormat("yy-MMM-dd:hh.mm.ss.SSS aaa");
    for (int i = 0; i < nrows; i++) {
      GenericData.Record record = new GenericRecordBuilder(schema)
              .set("mynull", null)
              .set("myboolean", i % 2 == 0)
              .set("myint", 1 + i)
              .set("mylong", 2L + i)
              .set("myfloat", 3.1f + i)
              .set("mydouble", 4.1 + i)
              .set("mydate", format.format(new Date(date.getTime() - (i * 1000 * 3600))))
              .set("myuuid", UUID.randomUUID())
              .set("mystring", "hello world: " + i)
              .set("myenum", i % 2 == 0 ? "a" : "b")
              .build();
      writer.write(record);
    }
  } finally {
    writer.close();
  }
  return f;
}
 

import org.apache.avro.generic.GenericRecordBuilder; //导入依赖的package包/类
@Test
public void testEmptyArray() throws Exception {
  Schema schema = new Schema.Parser().parse(
      Resources.getResource("array.avsc").openStream());

  File tmp = File.createTempFile(getClass().getSimpleName(), ".tmp");
  tmp.deleteOnExit();
  tmp.delete();
  Path file = new Path(tmp.getPath());

  AvroParquetWriter<GenericRecord> writer =
      new AvroParquetWriter<GenericRecord>(file, schema);

  // Write a record with an empty array.
  List<Integer> emptyArray = new ArrayList<Integer>();
  GenericData.Record record = new GenericRecordBuilder(schema)
      .set("myarray", emptyArray).build();
  writer.write(record);
  writer.close();

  AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf, file);
  GenericRecord nextRecord = reader.read();

  assertNotNull(nextRecord);
  assertEquals(emptyArray, nextRecord.get("myarray"));
}
 

import org.apache.avro.generic.GenericRecordBuilder; //导入依赖的package包/类
@Test
public void testEmptyMap() throws Exception {
  Schema schema = new Schema.Parser().parse(
      Resources.getResource("map.avsc").openStream());

  File tmp = File.createTempFile(getClass().getSimpleName(), ".tmp");
  tmp.deleteOnExit();
  tmp.delete();
  Path file = new Path(tmp.getPath());

  AvroParquetWriter<GenericRecord> writer = 
      new AvroParquetWriter<GenericRecord>(file, schema);

  // Write a record with an empty map.
  ImmutableMap emptyMap = new ImmutableMap.Builder<String, Integer>().build();
  GenericData.Record record = new GenericRecordBuilder(schema)
      .set("mymap", emptyMap).build();
  writer.write(record);
  writer.close();

  AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf, file);
  GenericRecord nextRecord = reader.read();

  assertNotNull(nextRecord);
  assertEquals(emptyMap, nextRecord.get("mymap"));
}
 

import org.apache.avro.generic.GenericRecordBuilder; //导入依赖的package包/类
@Test(expected=RuntimeException.class)
public void testMapRequiredValueWithNull() throws Exception {
  Schema schema = Schema.createRecord("record1", null, null, false);
  schema.setFields(Lists.newArrayList(
      new Schema.Field("mymap", Schema.createMap(Schema.create(Schema.Type.INT)), null, null)));

  File tmp = File.createTempFile(getClass().getSimpleName(), ".tmp");
  tmp.deleteOnExit();
  tmp.delete();
  Path file = new Path(tmp.getPath());

  AvroParquetWriter<GenericRecord> writer =
      new AvroParquetWriter<GenericRecord>(file, schema);

  // Write a record with a null value
  Map<String, Integer> map = new HashMap<String, Integer>();
  map.put("thirty-four", 34);
  map.put("eleventy-one", null);
  map.put("one-hundred", 100);

  GenericData.Record record = new GenericRecordBuilder(schema)
      .set("mymap", map).build();
  writer.write(record);
}
 

import org.apache.avro.generic.GenericRecordBuilder; //导入依赖的package包/类
@Test
public void testMapWithUtf8Key() throws Exception {
  Schema schema = new Schema.Parser().parse(
      Resources.getResource("map.avsc").openStream());

  File tmp = File.createTempFile(getClass().getSimpleName(), ".tmp");
  tmp.deleteOnExit();
  tmp.delete();
  Path file = new Path(tmp.getPath());

  AvroParquetWriter<GenericRecord> writer = 
      new AvroParquetWriter<GenericRecord>(file, schema);

  // Write a record with a map with Utf8 keys.
  GenericData.Record record = new GenericRecordBuilder(schema)
      .set("mymap", ImmutableMap.of(new Utf8("a"), 1, new Utf8("b"), 2))
      .build();
  writer.write(record);
  writer.close();

  AvroParquetReader<GenericRecord> reader = new AvroParquetReader<GenericRecord>(testConf, file);
  GenericRecord nextRecord = reader.read();

  assertNotNull(nextRecord);
  assertEquals(ImmutableMap.of(str("a"), 1, str("b"), 2), nextRecord.get("mymap"));
}