Java Type.asGroupType方法代码示例

import org.apache.parquet.schema.Type; //导入方法依赖的package包/类
/**
 * Changes the list inner '$data$' vector name to 'element' in the schema
 */
private Type renameChildTypeToElement(Type childType) {
  if (childType.isPrimitive()) {
    PrimitiveType childPrimitiveType = childType.asPrimitiveType();
    return new PrimitiveType(childType.getRepetition(),
      childPrimitiveType.getPrimitiveTypeName(),
      childPrimitiveType.getTypeLength(),
      "element",
      childPrimitiveType.getOriginalType(),
      childPrimitiveType.getDecimalMetadata(),
      null);
  } else {
    GroupType childGroupType = childType.asGroupType();
    return new GroupType(childType.getRepetition(),
      "element",
      childType.getOriginalType(),
      childGroupType.getFields());
  }
}
 

import org.apache.parquet.schema.Type; //导入方法依赖的package包/类
SimpleGroupConverter(SimpleGroupConverter parent, int index, GroupType schema) {
  this.parent = parent;
  this.index = index;

  converters = new Converter[schema.getFieldCount()];

  for (int i = 0; i < converters.length; i++) {
    final Type type = schema.getType(i);
    if (type.isPrimitive()) {
      converters[i] = new SimplePrimitiveConverter(this, i);
    } else {
      converters[i] = new SimpleGroupConverter(this, i, type.asGroupType());
    }

  }
}
 

import org.apache.parquet.schema.Type; //导入方法依赖的package包/类
private Converter newConverter(List<TProtocol> events, Type type, ThriftField field) {
  switch (field.getType().getType()) {
  case LIST:
    return new ListConverter(events, type.asGroupType(), field);
  case SET:
    return new SetConverter(events, type.asGroupType(), field);
  case MAP:
    return new MapConverter(events, type.asGroupType(), field);
  case STRUCT:
    return new StructConverter(events, type.asGroupType(), field);
  case STRING:
    return new FieldStringConverter(events, field);
  case ENUM:
    return new FieldEnumConverter(events, field);
  default:
    return new FieldPrimitiveConverter(events, field);
  }
}
 

import org.apache.parquet.schema.Type; //导入方法依赖的package包/类
protected static Converter getConverterFromDescription(final Type type, final int index,
    final HiveGroupConverter parent) {
  if (type == null) {
    return null;
  }
  if (type.isPrimitive()) {
    return ETypeConverter.getNewConverter(type.asPrimitiveType().getPrimitiveTypeName().javaType,
        index, parent);
  } else {
    if (type.asGroupType().getRepetition() == Repetition.REPEATED) {
      return new ArrayWritableGroupConverter(type.asGroupType(), parent, index);
    } else {
      return new DataWritableGroupConverter(type.asGroupType(), parent, index);
    }
  }
}
 

import org.apache.parquet.schema.Type; //导入方法依赖的package包/类
private Converter newScalarConverter(ParentValueContainer pvc, Message.Builder parentBuilder, Descriptors.FieldDescriptor fieldDescriptor, Type parquetType) {

    JavaType javaType = fieldDescriptor.getJavaType();

    switch (javaType) {
      case STRING: return new ProtoStringConverter(pvc);
      case FLOAT: return new ProtoFloatConverter(pvc);
      case DOUBLE: return new ProtoDoubleConverter(pvc);
      case BOOLEAN: return new ProtoBooleanConverter(pvc);
      case BYTE_STRING: return new ProtoBinaryConverter(pvc);
      case ENUM: return new ProtoEnumConverter(pvc, fieldDescriptor);
      case INT: return new ProtoIntConverter(pvc);
      case LONG: return new ProtoLongConverter(pvc);
      case MESSAGE: {
        Message.Builder subBuilder = parentBuilder.newBuilderForField(fieldDescriptor);
        return new ProtoMessageConverter(pvc, subBuilder, parquetType.asGroupType());
      }
    }

    throw new UnsupportedOperationException(String.format("Cannot convert type: %s" +
            " (Parquet type: %s) ", javaType, parquetType));
  }
 

import org.apache.parquet.schema.Type; //导入方法依赖的package包/类
private FieldWriter createWriter(Descriptors.FieldDescriptor fieldDescriptor, Type type) {

      switch (fieldDescriptor.getJavaType()) {
        case STRING: return new StringWriter() ;
        case MESSAGE: return new MessageWriter(fieldDescriptor.getMessageType(), type.asGroupType());
        case INT: return new IntWriter();
        case LONG: return new LongWriter();
        case FLOAT: return new FloatWriter();
        case DOUBLE: return new DoubleWriter();
        case ENUM: return new EnumWriter();
        case BOOLEAN: return new BooleanWriter();
        case BYTE_STRING: return new BinaryWriter();
      }

      return unknownType(fieldDescriptor);//should not be executed, always throws exception.
    }
 

import org.apache.parquet.schema.Type; //导入方法依赖的package包/类
public AvroArrayConverter(ParentValueContainer parent, GroupType type,
    Schema avroSchema, GenericData model) {
  this.parent = parent;
  this.avroSchema = avroSchema;
  Schema elementSchema = AvroSchemaConverter
      .getNonNull(avroSchema.getElementType());
  Type repeatedType = type.getType(0);
  // always determine whether the repeated type is the element type by
  // matching it against the element schema.
  if (AvroRecordConverter.isElementType(repeatedType, elementSchema)) {
    // the element type is the repeated type (and required)
    converter = newConverter(elementSchema, repeatedType, model, new ParentValueContainer() {
      @Override
      @SuppressWarnings("unchecked")
      public void add(Object value) {
        array.add(value);
      }
    });
  } else {
    // the element is wrapped in a synthetic group and may be optional
    converter = new ElementConverter(repeatedType.asGroupType(), elementSchema, model);
  }
}
 

import org.apache.parquet.schema.Type; //导入方法依赖的package包/类
public AvroUnionConverter(ParentValueContainer parent, Type parquetSchema,
                          Schema avroSchema, GenericData model) {
  this.parent = parent;
  GroupType parquetGroup = parquetSchema.asGroupType();
  this.memberConverters = new Converter[ parquetGroup.getFieldCount()];

  int parquetIndex = 0;
  for (int index = 0; index < avroSchema.getTypes().size(); index++) {
    Schema memberSchema = avroSchema.getTypes().get(index);
    if (!memberSchema.getType().equals(Schema.Type.NULL)) {
      Type memberType = parquetGroup.getType(parquetIndex);
      memberConverters[parquetIndex] = newConverter(memberSchema, memberType, model, new ParentValueContainer() {
        @Override
        public void add(Object value) {
          Preconditions.checkArgument(memberValue==null, "Union is resolving to more than one type");
          memberValue = value;
        }
      });
      parquetIndex++; // Note for nulls the parquetIndex id not increased
    }
  }
}
 

import org.apache.parquet.schema.Type; //导入方法依赖的package包/类
public AvroCollectionConverter(ParentValueContainer parent, GroupType type,
                               Schema avroSchema, GenericData model,
                               Class<?> containerClass) {
  this.parent = parent;
  this.avroSchema = avroSchema;
  this.containerClass = containerClass;
  Schema elementSchema = AvroSchemaConverter.getNonNull(avroSchema.getElementType());
  Type repeatedType = type.getType(0);
  // always determine whether the repeated type is the element type by
  // matching it against the element schema.
  if (isElementType(repeatedType, elementSchema)) {
    // the element type is the repeated type (and required)
    converter = newConverter(elementSchema, repeatedType, model, new ParentValueContainer() {
      @Override
      @SuppressWarnings("unchecked")
      public void add(Object value) {
        container.add(value);
      }
    });
  } else {
    // the element is wrapped in a synthetic group and may be optional
    converter = new ElementConverter(repeatedType.asGroupType(), elementSchema, model);
  }
}
 

import org.apache.parquet.schema.Type; //导入方法依赖的package包/类
public AvroArrayConverter(ParentValueContainer parent, GroupType type,
                          Schema avroSchema, GenericData model,
                          Class<?> arrayClass) {
  this.parent = parent;
  this.avroSchema = avroSchema;

  Preconditions.checkArgument(arrayClass.isArray(),
      "Cannot convert non-array: " + arrayClass.getName());
  this.elementClass = arrayClass.getComponentType();

  ParentValueContainer setter = createSetterAndContainer();
  Schema elementSchema = this.avroSchema.getElementType();
  Type repeatedType = type.getType(0);

  // always determine whether the repeated type is the element type by
  // matching it against the element schema.
  if (isElementType(repeatedType, elementSchema)) {
    // the element type is the repeated type (and required)
    converter = newConverter(elementSchema, repeatedType, model, elementClass, setter);
  } else {
    // the element is wrapped in a synthetic group and may be optional
    converter = new ArrayElementConverter(
        repeatedType.asGroupType(), elementSchema, model, setter);
  }
}
 

import org.apache.parquet.schema.Type; //导入方法依赖的package包/类
public AvroUnionConverter(ParentValueContainer parent, Type parquetSchema,
                          Schema avroSchema, GenericData model) {
  super(parent);
  GroupType parquetGroup = parquetSchema.asGroupType();
  this.memberConverters = new Converter[ parquetGroup.getFieldCount()];

  int parquetIndex = 0;
  for (int index = 0; index < avroSchema.getTypes().size(); index++) {
    Schema memberSchema = avroSchema.getTypes().get(index);
    if (!memberSchema.getType().equals(Schema.Type.NULL)) {
      Type memberType = parquetGroup.getType(parquetIndex);
      memberConverters[parquetIndex] = newConverter(memberSchema, memberType, model, new ParentValueContainer() {
        @Override
        public void add(Object value) {
          Preconditions.checkArgument(
              AvroUnionConverter.this.memberValue == null,
              "Union is resolving to more than one type");
          memberValue = value;
        }
      });
      parquetIndex++; // Note for nulls the parquetIndex id not increased
    }
  }
}
 

import org.apache.parquet.schema.Type; //导入方法依赖的package包/类
@Override
protected void addChildConverter(OutputMutator mutator, List<Field> arrowSchema, Iterator<SchemaPath> colIterator, Type type, Function<String, String> childNameResolver) {
  final String nameForChild = "inner";
  if (type.isPrimitive()) {
    converters.add( getConverterForType(nameForChild, type.asPrimitiveType()));
  } else {
    final GroupType groupType = type.asGroupType();
    Collection<SchemaPath> c = Lists.newArrayList(colIterator);
    if (arrowSchema != null) {
      converters.add( groupConverterFromArrowSchema(nameForChild, "$data$", groupType, c));
    } else {
      converters.add( defaultGroupConverter(mutator, groupType, nameForChild, c, null));
    }
  }
}
 

import org.apache.parquet.schema.Type; //导入方法依赖的package包/类
private Converter createConverter(Type field) {
  OriginalType otype = field.getOriginalType();

  if (field.isPrimitive()) {
    if (otype != null) {
      switch (otype) {
        case MAP: break;
        case LIST: break;
        case UTF8: return new StringConverter(field.getName());
        case MAP_KEY_VALUE: break;
        case ENUM: break;
        case DECIMAL:
          int scale = field.asPrimitiveType().getDecimalMetadata().getScale();
          return new DecimalConverter(field.getName(), scale);
      }
    }

    return new SimplePrimitiveConverter(field.getName());
  }

  GroupType groupType = field.asGroupType();
  if (otype != null) {
    switch (otype) {
      case MAP: return new SimpleMapRecordConverter(groupType, field.getName(), this);
      case LIST: return new SimpleListRecordConverter(groupType, field.getName(), this);
    }
  }
  return new SimpleRecordConverter(groupType, field.getName(), this);
}
 

import org.apache.parquet.schema.Type; //导入方法依赖的package包/类
private PrimitiveConverter getPrimitiveConverter(ColumnDescriptor path) {
  Type currentType = schema;
  Converter currentConverter = recordConverter;
  for (String fieldName : path.getPath()) {
    final GroupType groupType = currentType.asGroupType();
    int fieldIndex = groupType.getFieldIndex(fieldName);
    currentType = groupType.getType(fieldName);
    currentConverter = currentConverter.asGroupConverter().getConverter(fieldIndex);
  }
  PrimitiveConverter converter = currentConverter.asPrimitiveConverter();
  return converter;
}
 

import org.apache.parquet.schema.Type; //导入方法依赖的package包/类
/**
 * Will validate the structure of the list
 * @param list the Parquet List
 */
public List3Levels(GroupType list) {
  if (list.getOriginalType() != OriginalType.LIST || list.getFields().size() != 1) {
    throw new IllegalArgumentException("invalid list type: " + list);
  }
  this.list = list;
  Type repeatedField = list.getFields().get(0);
  if (repeatedField.isPrimitive() || !repeatedField.isRepetition(REPEATED) || repeatedField.asGroupType().getFields().size() != 1) {
    throw new IllegalArgumentException("invalid list type: " + list);
  }
  this.repeated = repeatedField.asGroupType();
  this.element = repeated.getFields().get(0);
}