Java Type类代码示例

import org.apache.parquet.schema.Type; //导入依赖的package包/类
/**
 * Converts {@link ColumnDescriptor} to {@link SchemaPath} and converts any parquet LOGICAL LIST to something
 * the execution engine can understand (removes the extra 'list' and 'element' fields from the name)
 */
private static SchemaPath convertColumnDescriptor(final MessageType schema, final ColumnDescriptor columnDescriptor) {
  List<String> path = Lists.newArrayList(columnDescriptor.getPath());

  // go through the path and find all logical lists
  int index = 0;
  Type type = schema;
  while (!type.isPrimitive()) { // don't bother checking the last element in the path as it is a primitive type
    type = type.asGroupType().getType(path.get(index));
    if (type.getOriginalType() == OriginalType.LIST && LogicalListL1Converter.isSupportedSchema(type.asGroupType())) {
      // remove 'list'
      type = type.asGroupType().getType(path.get(index+1));
      path.remove(index+1);
      // remove 'element'
      type = type.asGroupType().getType(path.get(index+1));
      path.remove(index+1);
    }
    index++;
  }

  String[] schemaColDesc = new String[path.size()];
  path.toArray(schemaColDesc);
  return SchemaPath.getCompoundPath(schemaColDesc);
}
 

import org.apache.parquet.schema.Type; //导入依赖的package包/类
/**
 * Changes the list inner '$data$' vector name to 'element' in the schema
 */
private Type renameChildTypeToElement(Type childType) {
  if (childType.isPrimitive()) {
    PrimitiveType childPrimitiveType = childType.asPrimitiveType();
    return new PrimitiveType(childType.getRepetition(),
      childPrimitiveType.getPrimitiveTypeName(),
      childPrimitiveType.getTypeLength(),
      "element",
      childPrimitiveType.getOriginalType(),
      childPrimitiveType.getDecimalMetadata(),
      null);
  } else {
    GroupType childGroupType = childType.asGroupType();
    return new GroupType(childType.getRepetition(),
      "element",
      childType.getOriginalType(),
      childGroupType.getFields());
  }
}
 

import org.apache.parquet.schema.Type; //导入依赖的package包/类
public ParquetValueConverter(GroupType schema, ParentContainerUpdater updater)
{
    super(updater);

    ArrayList<String> fieldNames = new ArrayList<>();
    for (Type type : schema.getFields()) {
        fieldNames.add(type.getName());
    }
    this.currentMap = new InternalMap(fieldNames);

    this.fieldConverters = new Converter[schema.getFieldCount()];
    int i = 0;
    for (Type field : schema.getFields()) {
        InternalMapUpdater update = new InternalMapUpdater(currentMap, i);
        fieldConverters[i++] = newFieldConverter(field, update);
    }
}
 

import org.apache.parquet.schema.Type; //导入依赖的package包/类
private Converter newFieldConverter(Type parquetType, ParentContainerUpdater updater)
{
    if (parquetType.isRepetition(Type.Repetition.REPEATED) && parquetType.getOriginalType() != OriginalType.LIST) {
        // A repeated field that is neither contained by a `LIST`- or `MAP`-annotated group nor
        // annotated by `LIST` or `MAP` should be interpreted as a required list of required
        // elements where the element type is the type of the field.
        if (parquetType.isPrimitive()) {
            return new RepeatedPrimitiveConverter(parquetType, updater);
        }
        else {
            return new RepeatedGroupConverter(parquetType, updater);
        }
    }
    else {
        return newConverter(parquetType, updater);
    }
}
 

import org.apache.parquet.schema.Type; //导入依赖的package包/类
private ParquetArrayConverter(GroupType schema, ParentContainerUpdater updater)
{
    super(updater);

    Type repeatedType = schema.getType(0);
    if (isElementType(repeatedType, schema.getName())) {
        // If the repeated field corresponds to the element type, creates a new converter using the
        // type of the repeated field.
        elementConverter = newConverter(repeatedType, new ParentContainerUpdater.Noop() {
            @Override
            public void set(Value value)
            {
                ParquetArrayConverter.this.currentArray.add(value);
            }
        });
    }
    else {
        // If the repeated field corresponds to the syntactic group in the standard 3-level Parquet
        // LIST layout, creates a new converter using the only child field of the repeated field.
        elementConverter = new ElementConverter(repeatedType.asGroupType().getType(0));
    }
}
 

import org.apache.parquet.schema.Type; //导入依赖的package包/类
public RepeatedPrimitiveConverter(Type parquetType, final ParentContainerUpdater parentUpdater)
{
    this.updater = new ParentContainerUpdater.Noop() {
        @Override
        public void start()
        {
            RepeatedPrimitiveConverter.this.currentArray = new ArrayList<>();
        }

        @Override
        public void end()
        {
            parentUpdater.set(ValueFactory.newArray(RepeatedPrimitiveConverter.this.currentArray));
        }

        @Override
        public void set(Value value)
        {
            RepeatedPrimitiveConverter.this.currentArray.add(value);
        }
    };

    this.elementConverter = newConverter(parquetType, getUpdater()).asPrimitiveConverter();
}
 

import org.apache.parquet.schema.Type; //导入依赖的package包/类
public RepeatedGroupConverter(Type parquetType, final ParentContainerUpdater parentUpdater)
{
    this.updater = new ParentContainerUpdater.Noop() {
        @Override
        public void start()
        {
            RepeatedGroupConverter.this.currentArray = new ArrayList<>();
        }

        @Override
        public void end()
        {
            parentUpdater.set(ValueFactory.newArray(RepeatedGroupConverter.this.currentArray));
        }

        @Override
        public void set(Value value)
        {
            RepeatedGroupConverter.this.currentArray.add(value);
        }
    };

    this.elementConverter = newConverter(parquetType, getUpdater()).asGroupConverter();
}
 

import org.apache.parquet.schema.Type; //导入依赖的package包/类
private ColTypeInfo getColTypeInfo(MessageType schema, Type type, String[] path, int depth) {
  if (type.isPrimitive()) {
    PrimitiveType primitiveType = (PrimitiveType) type;
    int precision = 0;
    int scale = 0;
    if (primitiveType.getDecimalMetadata() != null) {
      precision = primitiveType.getDecimalMetadata().getPrecision();
      scale = primitiveType.getDecimalMetadata().getScale();
    }

    int repetitionLevel = schema.getMaxRepetitionLevel(path);
    int definitionLevel = schema.getMaxDefinitionLevel(path);

    return new ColTypeInfo(type.getOriginalType(), precision, scale, repetitionLevel, definitionLevel);
  }
  Type t = ((GroupType) type).getType(path[depth]);
  return getColTypeInfo(schema, t, path, depth + 1);
}
 

import org.apache.parquet.schema.Type; //导入依赖的package包/类
private Type getType(MaterializedField field) {
  MinorType minorType = field.getType().getMinorType();
  DataMode dataMode = field.getType().getMode();
  switch(minorType) {
    case MAP:
      List<Type> types = Lists.newArrayList();
      for (MaterializedField childField : field.getChildren()) {
        types.add(getType(childField));
      }
      return new GroupType(dataMode == DataMode.REPEATED ? Repetition.REPEATED : Repetition.OPTIONAL, field.getName(), types);
    case LIST:
      throw new UnsupportedOperationException("Unsupported type " + minorType);
    default:
      return getPrimitiveType(field);
  }
}
 

import org.apache.parquet.schema.Type; //导入依赖的package包/类
private void writeObject(final Type type, final Object object, final int index) throws SerialisationException {
    if (null != object) {
        final String fieldName = type.getName();
        if (type.isPrimitive()) {
            recordConsumer.startField(fieldName, index);
            if (object instanceof Object[]) {
                for (final Object innerObject : (Object[]) object) {
                    writePrimitive(innerObject);
                }
            } else {
                writePrimitive(object);
            }
            recordConsumer.endField(fieldName, index);
        } else {
            final String originalType = type.getOriginalType().name();
            if ("MAP".equals(originalType)) {
                writeMap(fieldName, index, (Map<Object, Object>) object, type);
            } else if ("LIST".equals(originalType)) {
                writeList(fieldName, index, object, type);
            } else {
                throw new SerialisationException("Could not write object " + object.toString() + " with type " + type.toString());
            }
        }
    }
}
 

import org.apache.parquet.schema.Type; //导入依赖的package包/类
private void writeList(final String fieldName, final int index, final Object object, final Type type) throws SerialisationException {

        if (object instanceof List) {
            if (!((List) object).isEmpty()) {
                recordConsumer.startField(fieldName, index);
                recordConsumer.startGroup();
                recordConsumer.startField(LIST, 0);
                recordConsumer.startGroup();
                writeObject(type.asGroupType().getType(0).asGroupType().getType(0), ((List) object).toArray(), 0);
                recordConsumer.endGroup();
                recordConsumer.endField(LIST, 0);
                recordConsumer.endGroup();
                recordConsumer.endField(fieldName, index);
            }
        } else if (object instanceof Object[]) {
            recordConsumer.startField(fieldName, index);
            recordConsumer.startGroup();
            recordConsumer.startField(LIST, 0);
            recordConsumer.startGroup();
            writeObject(type.asGroupType().getType(0).asGroupType().getType(0), object, 0);
            recordConsumer.endGroup();
            recordConsumer.endField(LIST, 0);
            recordConsumer.endGroup();
            recordConsumer.endField(fieldName, index);
        }
    }
 

import org.apache.parquet.schema.Type; //导入依赖的package包/类
private Map<Integer, Converter> buildFieldToConverter(final GroupType schema) {
    final Map<Integer, Converter> fieldToConverter = new HashMap<>(fieldCount);
    int i = 0;
    for (final Type field : schema.getFields()) {
        final String[] newColumnPath = new String[columnPath.length + 1];
        int j = 0;
        for (final String part : columnPath) {
            newColumnPath[j] = part;
            j++;
        }
        newColumnPath[j] = field.getName();
        if (field.isPrimitive()) {
            fieldToConverter.put(i, new PrimitiveConverter(parquetColumnToObject, field.asPrimitiveType().getPrimitiveTypeName().javaType.getSimpleName(), newColumnPath, field.getOriginalType()));
        } else {
            fieldToConverter.put(i, new BypassGroupConverter(parquetColumnToObject, field.asGroupType(), newColumnPath));
        }
        i++;
    }
    return fieldToConverter;
}
 

import org.apache.parquet.schema.Type; //导入依赖的package包/类
/**
 * Given a schema, check to see if it is a union of a null type and a regular schema,
 * and then return the non-null sub-schema. Otherwise, return the given schema.
 *
 * @param schema The schema to check
 * @return The non-null portion of a union schema, or the given schema
 */
public static Schema getNonNull(Schema schema) {
  if (schema.getType().equals(Schema.Type.UNION)) {
    List<Schema> schemas = schema.getTypes();
    if (schemas.size() == 2) {
      if (schemas.get(0).getType().equals(Schema.Type.NULL)) {
        return schemas.get(1);
      } else if (schemas.get(1).getType().equals(Schema.Type.NULL)) {
        return schemas.get(0);
      } else {
        return schema;
      }
    } else {
      return schema;
    }
  } else {
    return schema;
  }
}
 

import org.apache.parquet.schema.Type; //导入依赖的package包/类
private Schema convertFields(String name, List<Type> parquetFields) {
  List<Schema.Field> fields = new ArrayList<Schema.Field>();
  for (Type parquetType : parquetFields) {
    Schema fieldSchema = convertField(parquetType);
    if (parquetType.isRepetition(REPEATED)) {
      throw new UnsupportedOperationException("REPEATED not supported outside LIST or MAP. Type: " + parquetType);
    } else if (parquetType.isRepetition(Type.Repetition.OPTIONAL)) {
      fields.add(new Schema.Field(
        parquetType.getName(), optional(fieldSchema), null, NullNode.getInstance()));
    } else { // REQUIRED
      fields.add(new Schema.Field(parquetType.getName(), fieldSchema, null, null));
    }
  }
  Schema schema = Schema.createRecord(name, null, null, false);
  schema.setFields(fields);
  return schema;
}
 

import org.apache.parquet.schema.Type; //导入依赖的package包/类
private void writeRecordFields(GroupType schema, Schema tajoSchema,
                               Tuple tuple) {
  List<Type> fields = schema.getFields();
  // Parquet ignores Tajo NULL_TYPE columns, so the index may differ.
  int index = 0;
  for (int tajoIndex = 0; tajoIndex < tajoSchema.size(); ++tajoIndex) {
    Column column = tajoSchema.getColumn(tajoIndex);
    if (column.getDataType().getType() == TajoDataTypes.Type.NULL_TYPE) {
      continue;
    }
    Type fieldType = fields.get(index);
    if (!tuple.isBlankOrNull(tajoIndex)) {
      recordConsumer.startField(fieldType.getName(), index);
      writeValue(column, tuple, tajoIndex);
      recordConsumer.endField(fieldType.getName(), index);
    } else if (fieldType.isRepetition(Type.Repetition.REQUIRED)) {
      throw new RuntimeException("Null-value for required field: " +
          column.getSimpleName());
    }
    ++index;
  }
}