Java GenericRecord類代碼示例

import org.apache.avro.generic.GenericRecord; //導入依賴的package包/類
private static void applyMapDiff(Schema.Field field, GenericRecord avroObj, GenericRecord fieldsValue, Map<Object, Object> modifiedObj, Object key) throws IOException {
    Map<String, Object> changedKeys = ((MapDiff) fieldsValue).getChangedKeys();

    for (String changedKey : changedKeys.keySet()) {
        Class<?> clazz = changedKeys.get(changedKey).getClass();

        if (clazz.isAssignableFrom(PrimitiveDiff.class)) {
            AvroDiffPrimitive.applyPrimitiveDiff(field, avroObj, changedKeys.get(changedKey), changedKeys, changedKey);
            modifiedObj.put(key, changedKeys);
        } else if (clazz.isAssignableFrom(MapDiff.class)) {
            AvroDiffMap.applyMapDiff(field, avroObj, (GenericRecord) changedKeys.get(changedKey), Maps.newHashMap(changedKeys), changedKey);
        } else if (clazz.isAssignableFrom(ArrayDiff.class)) {
            AvroDiffArray.applyArrayDiff(field, avroObj, (GenericRecord) changedKeys.get(changedKey), null);
        } else if (clazz.isAssignableFrom(RecordDiff.class)) {
            Object avroField = ((Map) avroObj.get(field.pos())).get(key);
            GenericRecord genericRecord = AvroDiff.applyDiff((GenericRecord) ((Map) avroField).get(changedKey), (RecordDiff) changedKeys.get(changedKey),
                    ((GenericRecord) ((Map) avroField).get(changedKey)).getSchema());
            ((Map) avroField).put(changedKey, genericRecord);
            modifiedObj.put(key, avroField);
        }
    }
}
 

import org.apache.avro.generic.GenericRecord; //導入依賴的package包/類
@Override
public Object write( final Object obj ) throws IOException{
  GenericRecord record = new GenericData.Record( avroSchema );
  if( ! ( obj instanceof Map ) ){
    return record;
  }

  Map<Object,Object> mapObj = (Map<Object,Object>)obj;

  for( KeyAndFormatter childFormatter : childContainer ){
    childFormatter.clear();
    record.put( childFormatter.getName() , childFormatter.get( mapObj ) );
  }

  return record;
}
 

import org.apache.avro.generic.GenericRecord; //導入依賴的package包/類
public static List processRecordY(CSVPrinter printer, GenericRecord record, List<Column> columns)
		throws IOException {
	List r = new ArrayList<>();
	columns.forEach(c -> {
		try {
			r.add(record.get(c.getField().name()));
		} catch (Exception e) {

			try {
				r.add(c.getDefaultValue());
			} catch (Exception e2) {
				r.add("NULL");
			}
		}
	});

	printer.printRecord(r);
	printer.flush();
	return r;
}
 

import org.apache.avro.generic.GenericRecord; //導入依賴的package包/類
/**
 * Import blob data that is smaller than inline lob limit and compress with
 * deflate codec. Blob data should be encoded and saved as Avro bytes.
 * @throws IOException
 * @throws SQLException
 */
public void testBlobCompressedAvroImportInline()
    throws IOException, SQLException {
  String [] types = { getBlobType() };
  String expectedVal = "This is short BLOB data";
  String [] vals = { getBlobInsertStr(expectedVal) };

  createTableWithColTypes(types, vals);

  runImport(getArgv("--compression-codec", CodecMap.DEFLATE));

  Path outputFile = new Path(getTablePath(), "part-m-00000.avro");
  DataFileReader<GenericRecord> reader = read(outputFile);
  GenericRecord record = reader.next();

  // Verify that the data block of the Avro file is compressed with deflate
  // codec.
  assertEquals(CodecMap.DEFLATE,
      reader.getMetaString(DataFileConstants.CODEC));

  // Verify that all columns are imported correctly.
  ByteBuffer buf = (ByteBuffer) record.get(getColName(0));
  String returnVal = new String(buf.array());

  assertEquals(getColName(0), expectedVal, returnVal);
}
 

import org.apache.avro.generic.GenericRecord; //導入依賴的package包/類
@Test
public void shouldCreateAndApplyRecordDiff() throws IOException {
    map1.put("a", 3f);
    map1.put("c", true);
    map2.put("c", true);
    map2.put("b", 5l);

    list1.add("asf");
    list2.add("ddd");

    RecordDiff diff = AvroDiff.createDiff(recordSpecificRecord1, recordSpecificRecord2, recordSpecificRecord1.getSchema());
    GenericRecord modifiedRecord = AvroDiff.applyDiff(recordSpecificRecord1, diff, recordSpecificRecord1.getSchema());

    Assert.assertEquals(modifiedRecord, recordSpecificRecord2);
    Assert.assertNotEquals(modifiedRecord, recordSpecificRecord1);
}
 

import org.apache.avro.generic.GenericRecord; //導入依賴的package包/類
/**
 * Create a data file that gets exported to the db.
 * @param fileNum the number of the file (for multi-file export)
 * @param numRecords how many records to write to the file.
 */
protected void createParquetFile(int fileNum, int numRecords,
    ColumnGenerator... extraCols) throws IOException {

  String uri = "dataset:file:" + getTablePath();
  Schema schema = buildSchema(extraCols);
  DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
    .schema(schema)
    .format(Formats.PARQUET)
    .build();
  Dataset dataset = Datasets.create(uri, descriptor);
  DatasetWriter writer = dataset.newWriter();
  try {
    for (int i = 0; i < numRecords; i++) {
      GenericRecord record = new GenericData.Record(schema);
      record.put("id", i);
      record.put("msg", getMsgPrefix() + i);
      addExtraColumns(record, i, extraCols);
      writer.write(record);
    }
  } finally {
    writer.close();
  }
}
 

import org.apache.avro.generic.GenericRecord; //導入依賴的package包/類
public static ControlMessage fullDataPollMessage(GenericRecord record, String topologyId, ConsumerRecord<String, byte[]> consumerRecord) {
    ControlMessage message = new ControlMessage();
    message.setId(System.currentTimeMillis());
    message.setFrom(topologyId);
    message.setType(FULL_DATA_PULL_REQ);
    message.addPayload("topic", consumerRecord.topic());
    message.addPayload("DBUS_DATASOURCE_ID", Utils.getDatasource().getId());
    PairWrapper<String, Object> wrapper = BoltCommandHandlerHelper.convertAvroRecord(record, Constants.MessageBodyKey.noorderKeys);
    message.addPayload("OP_TS", wrapper.getProperties(Constants.MessageBodyKey.OP_TS).toString());
    message.addPayload("POS", wrapper.getProperties(Constants.MessageBodyKey.POS).toString());
    for (Pair<String,Object> pair : wrapper.getPairs()) {
        message.addPayload(pair.getKey(), pair.getValue());
    }

    return message;
}
 

import org.apache.avro.generic.GenericRecord; //導入依賴的package包/類
public static <T extends Object> PairWrapper<String, Object> convertAvroRecordUseBeforeMap(GenericRecord record, Set<T> noorderKeys) {
    Schema schema = record.getSchema();
    List<Schema.Field> fields = schema.getFields();
    PairWrapper<String, Object> wrapper = new PairWrapper<>();

    for (Schema.Field field : fields) {
        String key = field.name();
        Object value = record.get(key);
        // 分離存儲是否關心順序的key-value
        if (noorderKeys.contains(field.name())) {
            wrapper.addProperties(key, value);
        }
    }

    GenericRecord before = getFromRecord(MessageBodyKey.BEFORE, record);

    Map<String, Object> beforeMap = convert2map(before);

    for (Map.Entry<String, Object> entry : beforeMap.entrySet()) {
        if(!entry.getKey().endsWith(MessageBodyKey.IS_MISSING_SUFFIX)) {
            wrapper.addPair(new Pair<>(entry.getKey(), CharSequence.class.isInstance(entry.getValue())?entry.getValue().toString():entry.getValue()));
        }
    }

    return wrapper;
}
 

import org.apache.avro.generic.GenericRecord; //導入依賴的package包/類
private GenericRecord convertToAvroRecord(Schema avroRecordSchema, Object[] values) {
  // TODO can be improve to create once and reuse
  GenericRecord avroRec = new GenericData.Record(avroRecordSchema);
  List<ColumnConverterDescriptor> columnConverters = converterDescriptor.getColumnConverters();
  if (values.length != columnConverters.size()) {
    // mismatch schema
    // TODO better exception
    throw new RuntimeException("Expecting " + columnConverters.size() + " fields, received "
        + values.length + " values");
  }
  for (int i = 0; i < values.length; i++) {
    Object value = values[i];
    ColumnConverterDescriptor columnConverterDescriptor = columnConverters.get(i);
    Object valueToWrite = columnConverterDescriptor.getWritable(value);
    avroRec.put(columnConverterDescriptor.getColumnName(), valueToWrite);
  }
  return avroRec;
}
 

import org.apache.avro.generic.GenericRecord; //導入依賴的package包/類
@SuppressWarnings("unchecked")
@Override
public T deserialize(String topic, byte[] data) {
	try {
		T result = null;

		if (data != null) {
			LOGGER.debug("data='{}'", DatatypeConverter.printHexBinary(data));

			DatumReader<GenericRecord> datumReader = new SpecificDatumReader<>(
					targetType.newInstance().getSchema());
			Decoder decoder = DecoderFactory.get().binaryDecoder(data, null);

			result = (T) datumReader.read(null, decoder);
			LOGGER.debug("deserialized data='{}'", result);
		}
		return result;
	} catch (Exception ex) {
		throw new SerializationException(
				"Can't deserialize data '" + Arrays.toString(data) + "' from topic '" + topic + "'", ex);
	}
}
 

import org.apache.avro.generic.GenericRecord; //導入依賴的package包/類
public void testFirstUnderscoreInColumnName() throws IOException {
  String [] names = { "_NAME" };
  String [] types = { "INT" };
  String [] vals = { "1987" };
  createTableWithColTypesAndNames(names, types, vals);

  runImport(getOutputArgv(true, null));

  Schema schema = getSchema();
  assertEquals(Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());
  checkField(fields.get(0), "__NAME", Type.INT);

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertEquals("__NAME", 1987, record1.get("__NAME"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.apache.avro.generic.GenericRecord; //導入依賴的package包/類
private static Path createDataFile() throws IOException {
    File avroFile = File.createTempFile("test-", "." + FILE_EXTENSION);
    DatumWriter<GenericRecord> writer = new GenericDatumWriter<>(schema);
    try (DataFileWriter<GenericRecord> dataFileWriter = new DataFileWriter<>(writer)) {
        dataFileWriter.setFlushOnEveryBlock(true);
        dataFileWriter.setSyncInterval(32);
        dataFileWriter.create(schema, avroFile);

        IntStream.range(0, NUM_RECORDS).forEach(index -> {
            GenericRecord datum = new GenericData.Record(schema);
            datum.put(FIELD_INDEX, index);
            datum.put(FIELD_NAME, String.format("%d_name_%s", index, UUID.randomUUID()));
            datum.put(FIELD_SURNAME, String.format("%d_surname_%s", index, UUID.randomUUID()));
            try {
                OFFSETS_BY_INDEX.put(index, dataFileWriter.sync() - 16L);
                dataFileWriter.append(datum);
            } catch (IOException ioe) {
                throw new RuntimeException(ioe);
            }
        });
    }
    Path path = new Path(new Path(fsUri), avroFile.getName());
    fs.moveFromLocalFile(new Path(avroFile.getAbsolutePath()), path);
    return path;
}
 

import org.apache.avro.generic.GenericRecord; //導入依賴的package包/類
public void testOverrideTypeMapping() throws IOException {
  String [] types = { "INT" };
  String [] vals = { "10" };
  createTableWithColTypes(types, vals);

  String [] extraArgs = { "--map-column-java", "DATA_COL0=String"};
  runImport(getOutputArgv(true, extraArgs));

  Schema schema = getSchema();
  assertEquals(Type.RECORD, schema.getType());
  List<Field> fields = schema.getFields();
  assertEquals(types.length, fields.size());
  checkField(fields.get(0), "DATA_COL0", Type.STRING);

  DatasetReader<GenericRecord> reader = getReader();
  try {
    assertTrue(reader.hasNext());
    GenericRecord record1 = reader.next();
    assertEquals("DATA_COL0", "10", record1.get("DATA_COL0"));
    assertFalse(reader.hasNext());
  } finally {
    reader.close();
  }
}
 

import org.apache.avro.generic.GenericRecord; //導入依賴的package包/類
@Test
public void testTimedFileRolling()
    throws EventDeliveryException, InterruptedException {
  // use a new roll interval
  config.put("kite.rollInterval", "1"); // in seconds

  DatasetSink sink = sink(in, config);

  Dataset<GenericRecord> records = Datasets.load(FILE_DATASET_URI);

  // run the sink
  sink.start();
  sink.process();

  Assert.assertEquals("Should have committed", 0, remaining(in));

  Thread.sleep(1100); // sleep longer than the roll interval
  sink.process(); // rolling happens in the process method

  Assert.assertEquals(Sets.newHashSet(expected), read(records));

  // wait until the end to stop because it would close the files
  sink.stop();
}
 

import org.apache.avro.generic.GenericRecord; //導入依賴的package包/類
/**
 * Converts the avro binary data to the json format
 */
@Override
public XContentBuilder serialize(Event event) {
    XContentBuilder builder = null;
    try {
        if (datumReader != null) {
            Decoder decoder = new DecoderFactory().binaryDecoder(event.getBody(), null);
            GenericRecord data = datumReader.read(null, decoder);
            logger.trace("Record in event " + data);
            XContentParser parser = XContentFactory
                    .xContent(XContentType.JSON)
                    .createParser(NamedXContentRegistry.EMPTY, data.toString());
            builder = jsonBuilder().copyCurrentStructure(parser);
            parser.close();
        } else {
            logger.error("Schema File is not configured");
        }
    } catch (IOException e) {
        logger.error("Exception in parsing avro format data but continuing serialization to process further records",
                e.getMessage(), e);
    }
    return builder;
}