Java StructTypeInfo类代码示例

import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; //导入依赖的package包/类
public static OrcSerde getFromTypeInfo( final Configuration config , final TypeInfo typeInfo )throws IOException{
  ObjectInspector objectInspector = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo( typeInfo );
  if( !( typeInfo instanceof StructTypeInfo ) ){
    throw new IOException( "Input type info is not StructTypeInfo : " + typeInfo.toString() );
  }
  String columnsName = "";
  String columnsType = "";
  List<TypeInfo> typeInfoList = ( (StructTypeInfo)typeInfo ).getAllStructFieldTypeInfos();
  List<StructField> structField = (List<StructField>)( ( (StructObjectInspector)objectInspector ).getAllStructFieldRefs() );
  for( int i = 0 ; i < structField.size() ; i++ ){
    if( ! columnsName.isEmpty() ){
      columnsName = columnsName.concat( "," );
      columnsType = columnsType.concat( "," );
    }
    columnsName = columnsName.concat( structField.get(i).getFieldName() );
    columnsType = columnsType.concat( typeInfoList.get(i).toString() );
  }

  OrcSerde serde = new OrcSerde();
  Properties table = new Properties();
  table.setProperty( serdeConstants.LIST_COLUMNS , columnsName );
  table.setProperty( serdeConstants.LIST_COLUMN_TYPES , columnsType );
  serde.initialize( config , table );

  return serde;
}
 

import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; //导入依赖的package包/类
private StructTypeInfo getAllReadTypeInfo( final String columnNameProperty , final String columnTypeProperty ){
  ArrayList<TypeInfo> fieldTypes = TypeInfoUtils.getTypeInfosFromTypeString( columnTypeProperty );
  ArrayList<String> columnNames = new ArrayList<String>();
  if ( columnNameProperty != null && 0 < columnNameProperty.length() ) {
    String[] columnNameArray = columnNameProperty.split(",");
    for( int i = 0 ; i < columnNameArray.length ; i++ ){
      columnNames.add( columnNameArray[i] );
      filedIndexMap.put( columnNameArray[i] , i );
    }
  }
  StructTypeInfo rootType = new StructTypeInfo();

  rootType.setAllStructFieldNames( columnNames );
  rootType.setAllStructFieldTypeInfos( fieldTypes );

  return rootType;
}
 

import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; //导入依赖的package包/类
private StructTypeInfo getColumnProjectionTypeInfo( final String columnNameProperty , final String columnTypeProperty , final String projectionColumnNames ){
  Set<String> columnNameSet = new HashSet<String>();
  for( String columnName : projectionColumnNames.split(",") ){
    columnNameSet.add( columnName );
  }

  ArrayList<TypeInfo> fieldTypes = TypeInfoUtils.getTypeInfosFromTypeString( columnTypeProperty );
  String[] splitNames = columnNameProperty.split(",");

  ArrayList<String> projectionColumnNameList = new ArrayList<String>();
  ArrayList<TypeInfo> projectionFieldTypeList = new ArrayList<TypeInfo>();
  for( int i = 0 ; i < fieldTypes.size() ; i++ ){
    if( columnNameSet.contains( splitNames[i] ) ){
      projectionColumnNameList.add( splitNames[i] );
      projectionFieldTypeList.add( fieldTypes.get(i) );
    }
    filedIndexMap.put( splitNames[i] , i );
  }
  StructTypeInfo rootType = new StructTypeInfo();

  rootType.setAllStructFieldNames( projectionColumnNameList );
  rootType.setAllStructFieldTypeInfos( projectionFieldTypeList );

  return rootType;
}
 

import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; //导入依赖的package包/类
@Override
public void initialize( final Configuration conf, final Properties table , final Properties part ) throws SerDeException{
  LOG.info( table.toString() );
  if( part != null ){
    LOG.info( part.toString() );
  }
  String columnNameProperty = table.getProperty(serdeConstants.LIST_COLUMNS);
  String columnTypeProperty = table.getProperty(serdeConstants.LIST_COLUMN_TYPES);

  String projectionColumnNames = conf.get( ColumnProjectionUtils.READ_COLUMN_NAMES_CONF_STR , "" );

  StructTypeInfo rootType;
  if( projectionColumnNames.isEmpty() ){
    rootType = getAllReadTypeInfo( columnNameProperty , columnTypeProperty );
  }
  else{
    rootType = getColumnProjectionTypeInfo( columnNameProperty , columnTypeProperty , projectionColumnNames );
  }

  inspector = MDSObjectInspectorFactory.craeteObjectInspectorFromTypeInfo( rootType );
}
 

import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; //导入依赖的package包/类
public HiveStructSchema( final StructContainerField schema ) throws IOException{
  this.schema = schema;

  StructTypeInfo structSchema = new StructTypeInfo();
  ArrayList<String> childKey = new ArrayList<String>();
  ArrayList<TypeInfo> childTypeInfo = new ArrayList<TypeInfo>();
  for( String key : schema.getKeys() ){
    TypeInfo typeInfo = HiveSchemaFactory.getHiveSchema( schema.get( key ) );
    childKey.add( key );
    childTypeInfo.add( typeInfo );
  }
  structSchema.setAllStructFieldNames( childKey );
  structSchema.setAllStructFieldTypeInfos( childTypeInfo );

  hiveSchema = structSchema;
}
 

import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; //导入依赖的package包/类
@Override
public Type toMetacatType(final String type) {
    // Hack to fix presto "varchar" type coming in with no length which is required by Hive.
    final TypeInfo typeInfo = TypeInfoUtils.getTypeInfoFromTypeString(
        "varchar".equals(type.toLowerCase()) ? serdeConstants.STRING_TYPE_NAME : type);
    ObjectInspector oi = TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo);
    // The standard struct object inspector forces field names to lower case, however in Metacat we need to preserve
    // the original case of the struct fields so we wrap it with our wrapper to force the fieldNames to keep
    // their original case
    if (typeInfo.getCategory().equals(ObjectInspector.Category.STRUCT)) {
        final StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
        final StandardStructObjectInspector objectInspector = (StandardStructObjectInspector) oi;
        oi = new HiveTypeConverter.SameCaseStandardStructObjectInspector(
            structTypeInfo.getAllStructFieldNames(), objectInspector);
    }
    return getCanonicalType(oi);
}
 

import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; //导入依赖的package包/类
public static LazySimpleStructObjectInspector createStructObjectInspector(TypeInfo type,
			LazySerDeParameters serdeParams) {
		StructTypeInfo structTypeInfo = (StructTypeInfo) type;
		List<String> fieldNames = structTypeInfo.getAllStructFieldNames();
		List<TypeInfo> fieldTypeInfos = structTypeInfo.getAllStructFieldTypeInfos();
		List<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>(fieldTypeInfos.size());

		for (int i = 0; i < fieldTypeInfos.size(); i++) {
			fieldObjectInspectors.add(createObjectInspector(fieldTypeInfos.get(i), serdeParams));
		}

		 return LazyObjectInspectorFactory.getLazySimpleStructObjectInspector(
			 fieldNames, fieldObjectInspectors, null,
			 serdeParams.getSeparators()[1],
			 serdeParams, ObjectInspectorOptions.JAVA);

//		return ObjectInspectorFactory.getStandardStructObjectInspector(fieldNames, fieldObjectInspectors);
	}
 

import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; //导入依赖的package包/类
private Object deserializeStruct(StructTypeInfo type, Object data)
        throws SerDeException {
    if (!(data instanceof Map)) {
        throw new SerDeException("Value not of type map");
    }
    //noinspection unchecked
    Map<String, Object> map = (Map<String, Object>) data;

    List<String> fieldNames = type.getAllStructFieldNames();
    List<TypeInfo> fieldTypes = type.getAllStructFieldTypeInfos();

    // When deserializing a struct the returned value is a list of values in the same order as the field names.

    List<Object> values = Lists.newArrayListWithCapacity(fieldNames.size());
    for (int i=0; i < fieldNames.size(); i++) {
        Object rawValue = getRawValueOrNullIfAbsent(fieldNames.get(i), map);
        Object value = deserialize(fieldTypes.get(i), rawValue);
        values.add(value);
    }

    return values;
}
 

import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; //导入依赖的package包/类
public ArrayWritableObjectInspector(final StructTypeInfo rowTypeInfo) {

        typeInfo = rowTypeInfo;
        fieldNames = rowTypeInfo.getAllStructFieldNames();
        fieldInfos = rowTypeInfo.getAllStructFieldTypeInfos();
        fields = new ArrayList<StructField>(fieldNames.size());
        fieldsByName = new HashMap<String, StructFieldImpl>();

        for (int i = 0; i < fieldNames.size(); ++i) {
            final String name = fieldNames.get(i);
            final TypeInfo fieldInfo = fieldInfos.get(i);

            final StructFieldImpl field = new StructFieldImpl(name, getObjectInspector(fieldInfo), i);
            fields.add(field);
            fieldsByName.put(name, field);
        }
    }
 

import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; //导入依赖的package包/类
public static boolean isSupportedType(TypeInfo typeInfo)
{
    switch (typeInfo.getCategory()) {
        case PRIMITIVE:
            PrimitiveObjectInspector.PrimitiveCategory primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory();
            return getPrimitiveType(primitiveCategory) != null;
        case MAP:
            MapTypeInfo mapTypeInfo = checkType(typeInfo, MapTypeInfo.class, "typeInfo");
            return isSupportedType(mapTypeInfo.getMapKeyTypeInfo()) && isSupportedType(mapTypeInfo.getMapValueTypeInfo());
        case LIST:
            ListTypeInfo listTypeInfo = checkType(typeInfo, ListTypeInfo.class, "typeInfo");
            return isSupportedType(listTypeInfo.getListElementTypeInfo());
        case STRUCT:
            StructTypeInfo structTypeInfo = checkType(typeInfo, StructTypeInfo.class, "typeInfo");
            return structTypeInfo.getAllStructFieldTypeInfos().stream()
                    .allMatch(HiveType::isSupportedType);
    }
    return false;
}
 

import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; //导入依赖的package包/类
@Test
public void readFullyReadSchemaFromSplit() throws IOException {
  StructTypeInfo typeInfo = new StructTypeInfoBuilder()
      .add("a", TypeInfoFactory.stringTypeInfo)
      .add("b", TypeInfoFactory.stringTypeInfo)
      .build();
  CorcInputFormat.setTypeInfo(conf, typeInfo);
  CorcInputFormat.setConverterFactoryClass(conf, DefaultConverterFactory.class);

  RecordReader<NullWritable, Corc> reader = inputFormat.getRecordReader(split, conf, reporter);

  Corc corc = reader.createValue();

  reader.next(NullWritable.get(), corc);
  assertThat(corc.get("a"), is((Object) "A1"));
  assertThat(corc.get("b"), is((Object) "B1"));
  reader.close();
}
 

import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; //导入依赖的package包/类
@Test
public void readFullyDeclaredSchema() throws IOException {
  StructTypeInfo typeInfo = new StructTypeInfoBuilder()
      .add("a", TypeInfoFactory.stringTypeInfo)
      .add("b", TypeInfoFactory.stringTypeInfo)
      .build();
  CorcInputFormat.setTypeInfo(conf, typeInfo);
  CorcInputFormat.setSchemaTypeInfo(conf, typeInfo);
  CorcInputFormat.setConverterFactoryClass(conf, DefaultConverterFactory.class);

  RecordReader<NullWritable, Corc> reader = inputFormat.getRecordReader(split, conf, reporter);

  Corc corc = reader.createValue();

  reader.next(NullWritable.get(), corc);
  assertThat(corc.get("a"), is((Object) "A1"));
  assertThat(corc.get("b"), is((Object) "B1"));
  reader.close();
}
 

import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; //导入依赖的package包/类
static Fields newFields(StructTypeInfo structTypeInfo) {
  List<String> existingNames = structTypeInfo.getAllStructFieldNames();
  List<String> namesList = new ArrayList<>(existingNames.size());
  namesList.addAll(existingNames);
  String[] names = namesList.toArray(new String[namesList.size()]);

  List<TypeInfo> typeInfos = structTypeInfo.getAllStructFieldTypeInfos();
  Class<?>[] types = new Class[typeInfos.size()];
  for (int i = 0; i < types.length; i++) {
    Class<?> type = PRIMITIVES.get(typeInfos.get(i));
    if (type == null) {
      type = Object.class;
    }
    types[i] = type;
  }

  return new Fields(names, types);
}
 

import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; //导入依赖的package包/类
static StructTypeInfo newStructTypeInfo(Fields fields) {
  List<String> names = new ArrayList<>();
  List<TypeInfo> typeInfos = new ArrayList<>();

  for (int i = 0; i < fields.size(); i++) {
    String name = fields.get(i).toString();
    if (ROW_ID_NAME.equals(name)) {
      if (!fields.getTypeClass(i).equals(RecordIdentifier.class)) {
        throw new IllegalArgumentException(ROW_ID_NAME + " column is not of type "
            + RecordIdentifier.class.getSimpleName() + ". Found type: " + fields.getTypeClass(i));
      }
      continue;
    }
    names.add(name.toLowerCase());
    Class<?> type = fields.getTypeClass(i);
    if (type == null) {
      throw new IllegalArgumentException("Missing type information for field: " + name);
    }

    TypeInfo typeInfo = getTypeInfoFromClass(type);
    typeInfos.add(typeInfo);
  }

  return (StructTypeInfo) TypeInfoFactory.getStructTypeInfo(names, typeInfos);
}
 

import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo; //导入依赖的package包/类
private StructTypeInfo createTypeInfo() {
  return new StructTypeInfoBuilder()
      .add("a", TypeInfoFactory.stringTypeInfo)
      .add("b", TypeInfoFactory.booleanTypeInfo)
      .add("c", TypeInfoFactory.byteTypeInfo)
      .add("d", TypeInfoFactory.shortTypeInfo)
      .add("e", TypeInfoFactory.intTypeInfo)
      .add("f", TypeInfoFactory.longTypeInfo)
      .add("g", TypeInfoFactory.floatTypeInfo)
      .add("h", TypeInfoFactory.doubleTypeInfo)
      .add("i", TypeInfoFactory.timestampTypeInfo)
      .add("j", TypeInfoFactory.dateTypeInfo)
      .add("k", TypeInfoFactory.binaryTypeInfo)
      .add("l", TypeInfoFactory.decimalTypeInfo)
      .add("m", TypeInfoFactory.getListTypeInfo(TypeInfoFactory.intTypeInfo))
      .add("n", TypeInfoFactory.getMapTypeInfo(TypeInfoFactory.intTypeInfo, TypeInfoFactory.intTypeInfo))
      .add("o", new StructTypeInfoBuilder().add("a", TypeInfoFactory.intTypeInfo).build())
      .add("p", TypeInfoFactory.getUnionTypeInfo(Arrays.asList((TypeInfo) TypeInfoFactory.stringTypeInfo)))
      .build();
}