Java AvroData类代码示例

import io.confluent.connect.avro.AvroData; //导入依赖的package包/类
public static byte[] putRecords(Collection<SinkRecord> records, AvroData avroData) throws IOException {
  final DataFileWriter<Object> writer = new DataFileWriter<>(new GenericDatumWriter<>());
  ByteArrayOutputStream out = new ByteArrayOutputStream();
  Schema schema = null;
  for (SinkRecord record : records) {
    if (schema == null) {
      schema = record.valueSchema();
      org.apache.avro.Schema avroSchema = avroData.fromConnectSchema(schema);
      writer.create(avroSchema, out);
    }
    Object value = avroData.fromConnectData(schema, record.value());
    // AvroData wraps primitive types so their schema can be included. We need to unwrap
    // NonRecordContainers to just their value to properly handle these types
    if (value instanceof NonRecordContainer) {
      value = ((NonRecordContainer) value).getValue();
    }
    writer.append(value);
  }
  writer.flush();
  return out.toByteArray();
}
 

import io.confluent.connect.avro.AvroData; //导入依赖的package包/类
@Test
public void testRetrieveSchema() throws Exception {
  final TableId table = TableId.of("test", "kafka_topic");
  final String testTopic = "kafka-topic";
  final String testSubject = "kafka-topic-value";
  final String testAvroSchemaString =
      "{\"type\": \"record\", "
      + "\"name\": \"testrecord\", "
      + "\"fields\": [{\"name\": \"f1\", \"type\": \"string\"}]}";
  final SchemaMetadata testSchemaMetadata = new SchemaMetadata(1, 1, testAvroSchemaString);

  SchemaRegistryClient schemaRegistryClient = mock(SchemaRegistryClient.class);
  when(schemaRegistryClient.getLatestSchemaMetadata(testSubject)).thenReturn(testSchemaMetadata);

  SchemaRegistrySchemaRetriever testSchemaRetriever = new SchemaRegistrySchemaRetriever(
      schemaRegistryClient,
      new AvroData(0)
  );

  Schema expectedKafkaConnectSchema =
      SchemaBuilder.struct().field("f1", Schema.STRING_SCHEMA).name("testrecord").build();

  assertEquals(expectedKafkaConnectSchema, testSchemaRetriever.retrieveSchema(table, testTopic));
}
 

import io.confluent.connect.avro.AvroData; //导入依赖的package包/类
public TopicPartitionWriter(
    TopicPartition tp,
    Storage storage,
    RecordWriterProvider writerProvider,
    Partitioner partitioner,
    HdfsSinkConnectorConfig connectorConfig,
    SinkTaskContext context,
    AvroData avroData) {
  this(tp, storage, writerProvider, partitioner, connectorConfig, context, avroData, null, null, null, null, null);
}
 

import io.confluent.connect.avro.AvroData; //导入依赖的package包/类
protected void configureConnector() {
  connectorConfig = new HdfsSinkConnectorConfig(connectorProps);
  topicsDir = connectorConfig.getString(HdfsSinkConnectorConfig.TOPICS_DIR_CONFIG);
  logsDir = connectorConfig.getString(HdfsSinkConnectorConfig.LOGS_DIR_CONFIG);
  int schemaCacheSize = connectorConfig.getInt(HdfsSinkConnectorConfig.SCHEMA_CACHE_SIZE_CONFIG);
  avroData = new AvroData(schemaCacheSize);
}
 

import io.confluent.connect.avro.AvroData; //导入依赖的package包/类
@Override
public RecordWriter<SinkRecord> getRecordWriter(
    Configuration conf, final String fileName, SinkRecord record, final AvroData avroData)
    throws IOException {

  final Map<String, List<Object>> data = Data.getData();

  if (!data.containsKey(fileName)) {
    data.put(fileName, new LinkedList<>());
  }

  return new MemoryRecordWriter(fileName);
}
 

import io.confluent.connect.avro.AvroData; //导入依赖的package包/类
private int calcByteSize(List<SinkRecord> sinkRecords) throws IOException {
  ByteArrayOutputStream baos = new ByteArrayOutputStream();
  DataFileWriter<Object> writer = new DataFileWriter<>(new GenericDatumWriter<>());
  AvroData avroData = new AvroData(1);
  boolean writerInit = false;
  for(SinkRecord sinkRecord: sinkRecords){
    if(!writerInit){
      writer.create(avroData.fromConnectSchema(sinkRecord.valueSchema()), baos);
      writerInit = true;
    }
    writer.append(avroData.fromConnectData(sinkRecord.valueSchema(), sinkRecord.value()));
  }
  return baos.size();
}
 

import io.confluent.connect.avro.AvroData; //导入依赖的package包/类
@Override
public void configure(Map<String, String> properties) {
  SchemaRegistrySchemaRetrieverConfig config =
      new SchemaRegistrySchemaRetrieverConfig(properties);
  schemaRegistryClient =
      new CachedSchemaRegistryClient(config.getString(config.LOCATION_CONFIG), 0);
  avroData = new AvroData(config.getInt(config.AVRO_DATA_CACHE_SIZE_CONFIG));
}
 

import io.confluent.connect.avro.AvroData; //导入依赖的package包/类
public GenericRecordToStruct() {
    this.avroData = new AvroData(CACHE_SIZE);
}
 

import io.confluent.connect.avro.AvroData; //导入依赖的package包/类
public HiveUtil(HdfsSinkConnectorConfig connectorConfig, AvroData avroData, HiveMetaStore hiveMetaStore) {
  this.url = connectorConfig.getString(HdfsSinkConnectorConfig.HDFS_URL_CONFIG);
  this.topicsDir = connectorConfig.getString(HdfsSinkConnectorConfig.TOPICS_DIR_CONFIG);
  this.avroData = avroData;
  this.hiveMetaStore = hiveMetaStore;
}
 

import io.confluent.connect.avro.AvroData; //导入依赖的package包/类
public AvroHiveUtil(HdfsSinkConnectorConfig connectorConfig, AvroData avroData, HiveMetaStore hiveMetaStore) {
  super(connectorConfig, avroData, hiveMetaStore);
}
 

import io.confluent.connect.avro.AvroData; //导入依赖的package包/类
public AvroFileReader(AvroData avroData) {
  this.avroData = avroData;
}
 

import io.confluent.connect.avro.AvroData; //导入依赖的package包/类
public SchemaFileReader getSchemaFileReader(AvroData avroData) {
  return new AvroFileReader(avroData);
}
 

import io.confluent.connect.avro.AvroData; //导入依赖的package包/类
public HiveUtil getHiveUtil(HdfsSinkConnectorConfig config, AvroData avroData, HiveMetaStore hiveMetaStore) {
  return new AvroHiveUtil(config, avroData, hiveMetaStore);
}
 

import io.confluent.connect.avro.AvroData; //导入依赖的package包/类
public AvroData getAvroData() {
  return avroData;
}
 

import io.confluent.connect.avro.AvroData; //导入依赖的package包/类
public ParquetHiveUtil(HdfsSinkConnectorConfig connectorConfig, AvroData avroData, HiveMetaStore hiveMetaStore) {
  super(connectorConfig, avroData, hiveMetaStore);
}