Java Schema.Field方法代码示例

import org.apache.avro.Schema; //导入方法依赖的package包/类
private static void applyMapDiff(Schema.Field field, GenericRecord avroObj, GenericRecord fieldsValue, Map<Object, Object> modifiedObj, Object key) throws IOException {
    Map<String, Object> changedKeys = ((MapDiff) fieldsValue).getChangedKeys();

    for (String changedKey : changedKeys.keySet()) {
        Class<?> clazz = changedKeys.get(changedKey).getClass();

        if (clazz.isAssignableFrom(PrimitiveDiff.class)) {
            AvroDiffPrimitive.applyPrimitiveDiff(field, avroObj, changedKeys.get(changedKey), changedKeys, changedKey);
            modifiedObj.put(key, changedKeys);
        } else if (clazz.isAssignableFrom(MapDiff.class)) {
            AvroDiffMap.applyMapDiff(field, avroObj, (GenericRecord) changedKeys.get(changedKey), Maps.newHashMap(changedKeys), changedKey);
        } else if (clazz.isAssignableFrom(ArrayDiff.class)) {
            AvroDiffArray.applyArrayDiff(field, avroObj, (GenericRecord) changedKeys.get(changedKey), null);
        } else if (clazz.isAssignableFrom(RecordDiff.class)) {
            Object avroField = ((Map) avroObj.get(field.pos())).get(key);
            GenericRecord genericRecord = AvroDiff.applyDiff((GenericRecord) ((Map) avroField).get(changedKey), (RecordDiff) changedKeys.get(changedKey),
                    ((GenericRecord) ((Map) avroField).get(changedKey)).getSchema());
            ((Map) avroField).put(changedKey, genericRecord);
            modifiedObj.put(key, avroField);
        }
    }
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
protected SqoopRecord toSqoopRecord(GenericRecord record) throws IOException {
  Schema avroSchema = record.getSchema();
  for (Map.Entry<Writable, Writable> e : columnTypes.entrySet()) {
    String columnName = e.getKey().toString();
    String columnType = e.getValue().toString();
    String cleanedCol = ClassWriter.toIdentifier(columnName);
    Schema.Field field = getFieldIgnoreCase(avroSchema, cleanedCol);
    if (null == field) {
      throw new IOException("Cannot find field " + cleanedCol
          + " in Avro schema " + avroSchema);
    }

    Object avroObject = record.get(field.name());
    Object fieldVal = AvroUtil.fromAvro(avroObject, field.schema(), columnType);
    recordImpl.setField(cleanedCol, fieldVal);
  }
  return recordImpl;
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
public static <T extends Object> PairWrapper<String, Object> convertAvroRecordUseBeforeMap(GenericRecord record, Set<T> noorderKeys) {
    Schema schema = record.getSchema();
    List<Schema.Field> fields = schema.getFields();
    PairWrapper<String, Object> wrapper = new PairWrapper<>();

    for (Schema.Field field : fields) {
        String key = field.name();
        Object value = record.get(key);
        // 分离存储是否关心顺序的key-value
        if (noorderKeys.contains(field.name())) {
            wrapper.addProperties(key, value);
        }
    }

    GenericRecord before = getFromRecord(MessageBodyKey.BEFORE, record);

    Map<String, Object> beforeMap = convert2map(before);

    for (Map.Entry<String, Object> entry : beforeMap.entrySet()) {
        if(!entry.getKey().endsWith(MessageBodyKey.IS_MISSING_SUFFIX)) {
            wrapper.addPair(new Pair<>(entry.getKey(), CharSequence.class.isInstance(entry.getValue())?entry.getValue().toString():entry.getValue()));
        }
    }

    return wrapper;
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
@Test
public void testValidateSchemaPairAllFields() throws Exception {
  final List<Schema.Field> readerFields = list(
      new Schema.Field("oldfield1", INT_SCHEMA, null, null),
      new Schema.Field("oldfield2", STRING_SCHEMA, null, null));
  final Schema reader = Schema.createRecord(readerFields);
  final SchemaCompatibility.SchemaPairCompatibility expectedResult =
      new SchemaCompatibility.SchemaPairCompatibility(
          SchemaCompatibility.SchemaCompatibilityResult.compatible(),
          reader,
          WRITER_SCHEMA,
          SchemaCompatibility.READER_WRITER_COMPATIBLE_MESSAGE);

  // Test with all fields.
  assertEquals(expectedResult, checkReaderWriterCompatibility(reader, WRITER_SCHEMA));
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
/**
 * Creates the avro record schema
 * 
 * @param recordName - Name of record.
 * @param columnNames - Column/field names
 * @param schemas - Avro schema for each field/column
 * @return - Avro Record schema
 */
public static Schema createAvroRecordSchema(String recordName, String[] columnNames,
    Schema[] schemas) {
  Schema record = Schema.createRecord(recordName, null, null, false);
  // create fields
  List<Schema.Field> fields = new ArrayList<>();
  // String field
  for (int i = 0; i < columnNames.length; i++) {
    String fieldName = columnNames[i];
    Schema fieldType = schemas[i];
    Schema.Field field = new Schema.Field(fieldName, fieldType, null, null,
        ParquetColumnConverterDescriptor.DEFAULT_ORDER);
    fields.add(field);
  }
  record.setFields(fields);
  return record;
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
private static GenericRecord createGenericRecordWithSchema(Schema schema, GenericRecord avrObj){
    GenericRecordBuilder builder = new GenericRecordBuilder(schema);
    for (Schema.Field field : schema.getFields()){
        builder.set(field, avrObj.get(field.name()));
    }
    return builder.build();
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
public static void applyPrimitiveDiff(Schema.Field field, GenericRecord avroObj, Object fieldsValue, Object modifiedObj, Object key) throws IOException {
    ByteBuffer diffValue = ((PrimitiveDiff) fieldsValue).getDiffValue();
    Object newValue = ((PrimitiveDiff) fieldsValue).getNewValue();

    if (newValue != null) {
        if (key != null) {
            ((Map) modifiedObj).put(key, newValue);
        } else {
            ((GenericRecord) modifiedObj).put(field.pos(), newValue);
        }
    } else {
        Object avroObjField = avroObj.get(field.pos());
        byte[] avroObjFieldBytes = field.schema().getType().equals(Schema.Type.STRING) ? ((String) avroObjField).getBytes() : ((byte[]) avroObjField);
        byte[] avroObjFieldModifiedBytes = new GDiffPatcher().patch(avroObjFieldBytes, diffValue.array());
        if (key != null) {
            if (field.schema().getType().equals(Schema.Type.STRING)) {
                ((Map) modifiedObj).put(key, new String(avroObjFieldModifiedBytes, StandardCharsets.UTF_8));
            } else {
                ((Map) modifiedObj).put(key, avroObjFieldModifiedBytes);
            }
        } else {
            if (field.schema().getType().equals(Schema.Type.STRING)) {
                ((GenericRecord) modifiedObj).put(field.pos(), new String(avroObjFieldModifiedBytes, StandardCharsets.UTF_8));
            } else {
                ((GenericRecord) modifiedObj).put(field.pos(), avroObjFieldModifiedBytes);
            }
        }
    }
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
@Override
public Option<Model> read(Class<?> cls, Map<String, String> typeMap) {
  Schema schema = getSchema(cls);
  if (schema == null) {
    return Option.empty();
  }

  LinkedHashMap<String, ModelProperty> properties = new LinkedHashMap<>();
  for (Schema.Field field : schema.getFields()) {
    ModelProperty property = parseField(field);
    if (property == null) {
      LOGGER.debug(
          "Omitted field {} of schema {} from swagger docs", field.name(), schema.getName());
    } else {
      properties.update(getFieldName(field), property);
    }
  }

  return Option.apply(
      new Model(
          toName(cls),
          toName(cls),
          cls.getName(),
          properties,
          toDescriptionOpt(cls),
          Option.<String>empty(),
          Option.<String>empty(),
          JavaConversions.asScalaBuffer(Collections.<String>emptyList()).toList()));
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
private SqoopRecord toSqoopRecord(GenericRecord genericRecord) throws IOException {
  Schema avroSchema = genericRecord.getSchema();
  for (Schema.Field field : avroSchema.getFields()) {
    Pair<String, String> sqoopRecordField = sqoopRecordFields.get(field.name().toLowerCase());
    if (null == sqoopRecordField) {
      throw new IOException("Cannot find field '" + field.name() + "' in fields of user class"
          + sqoopRecordImpl.getClass().getName() + ". Fields are: "
          + Arrays.deepToString(sqoopRecordFields.values().toArray()));
    }
    Object avroObject = genericRecord.get(field.name());
    Object fieldVal = AvroUtil.fromAvro(avroObject, field.schema(), sqoopRecordField.value());
    sqoopRecordImpl.setField(sqoopRecordField.key(), fieldVal);
  }
  return sqoopRecordImpl;
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
public static <T extends Object> PairWrapper<String, Object> convertAvroRecord(GenericRecord record, Set<T> noorderKeys) {
    Schema schema = record.getSchema();
    List<Schema.Field> fields = schema.getFields();
    PairWrapper<String, Object> wrapper = new PairWrapper<>();

    for (Schema.Field field : fields) {
        String key = field.name();
        Object value = record.get(key);
        // 分离存储是否关心顺序的key-value
        if (noorderKeys.contains(field.name())) {
            wrapper.addProperties(key, value);
        }
    }

    GenericRecord before = getFromRecord(MessageBodyKey.BEFORE, record);
    GenericRecord after = getFromRecord(MessageBodyKey.AFTER, record);

    Map<String, Object> beforeMap = convert2map(before);
    Map<String, Object> afterMap = convert2map(after);

    // 覆盖before
    mergeMap(beforeMap, afterMap);

    for (Map.Entry<String, Object> entry : beforeMap.entrySet()) {
        if(!entry.getKey().endsWith(MessageBodyKey.IS_MISSING_SUFFIX)) {
            wrapper.addPair(new Pair<>(entry.getKey(), CharSequence.class.isInstance(entry.getValue())?entry.getValue().toString():entry.getValue()));
        }
    }

    return wrapper;
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
private static Map<String, Object> convert2map(GenericRecord record) {
    Map<String, Object> map = Maps.newHashMap();
    if (record != null) {
        List<Schema.Field> fields = record.getSchema().getFields();
        for (Schema.Field field : fields) {
            String key = field.name();
            map.put(key, record.get(key));
        }
    }
    return map;
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
@Test
public void testValidateSchemaPairMissingField() throws Exception {
  final List<Schema.Field> readerFields = list(
      new Schema.Field("oldfield1", INT_SCHEMA, null, null));
  final Schema reader = Schema.createRecord(readerFields);
  final SchemaCompatibility.SchemaPairCompatibility expectedResult =
      new SchemaCompatibility.SchemaPairCompatibility(
          SchemaCompatibility.SchemaCompatibilityResult.compatible(),
          reader,
          WRITER_SCHEMA,
          SchemaCompatibility.READER_WRITER_COMPATIBLE_MESSAGE);

  // Test omitting a field.
  assertEquals(expectedResult, checkReaderWriterCompatibility(reader, WRITER_SCHEMA));
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
@Test
public void testValidateSchemaPairMissingSecondField() throws Exception {
  final List<Schema.Field> readerFields = list(
      new Schema.Field("oldfield2", STRING_SCHEMA, null, null));
  final Schema reader = Schema.createRecord(readerFields);
  final SchemaCompatibility.SchemaPairCompatibility expectedResult =
      new SchemaCompatibility.SchemaPairCompatibility(
          SchemaCompatibility.SchemaCompatibilityResult.compatible(),
          reader,
          WRITER_SCHEMA,
          SchemaCompatibility.READER_WRITER_COMPATIBLE_MESSAGE);

  // Test omitting other field.
  assertEquals(expectedResult, checkReaderWriterCompatibility(reader, WRITER_SCHEMA));
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
public AvroRecordSchema( final StructContainerField schema ) throws IOException{
  this.schema = schema;

  List<Schema.Field> childList = new ArrayList<Schema.Field>();
  for( String key : schema.getKeys() ){
    childList.add( new Schema.Field( key , AvroSchemaFactory.getAvroSchema( schema.get( key ) ) , null , null ) );
  }
  avroSchema = Schema.createRecord( childList );
}
 

import org.apache.avro.Schema; //导入方法依赖的package包/类
private static Schema.Field getFieldIgnoreCase(Schema avroSchema,
    String fieldName) {
  for (Schema.Field field : avroSchema.getFields()) {
    if (field.name().equalsIgnoreCase(fieldName)) {
      return field;
    }
  }
  return null;
}