Java OriginalType.LIST属性代码示例

/**
 * Converts {@link ColumnDescriptor} to {@link SchemaPath} and converts any parquet LOGICAL LIST to something
 * the execution engine can understand (removes the extra 'list' and 'element' fields from the name)
 */
private static SchemaPath convertColumnDescriptor(final MessageType schema, final ColumnDescriptor columnDescriptor) {
  List<String> path = Lists.newArrayList(columnDescriptor.getPath());

  // go through the path and find all logical lists
  int index = 0;
  Type type = schema;
  while (!type.isPrimitive()) { // don't bother checking the last element in the path as it is a primitive type
    type = type.asGroupType().getType(path.get(index));
    if (type.getOriginalType() == OriginalType.LIST && LogicalListL1Converter.isSupportedSchema(type.asGroupType())) {
      // remove 'list'
      type = type.asGroupType().getType(path.get(index+1));
      path.remove(index+1);
      // remove 'element'
      type = type.asGroupType().getType(path.get(index+1));
      path.remove(index+1);
    }
    index++;
  }

  String[] schemaColDesc = new String[path.size()];
  path.toArray(schemaColDesc);
  return SchemaPath.getCompoundPath(schemaColDesc);
}
 

private Converter newFieldConverter(Type parquetType, ParentContainerUpdater updater)
{
    if (parquetType.isRepetition(Type.Repetition.REPEATED) && parquetType.getOriginalType() != OriginalType.LIST) {
        // A repeated field that is neither contained by a `LIST`- or `MAP`-annotated group nor
        // annotated by `LIST` or `MAP` should be interpreted as a required list of required
        // elements where the element type is the type of the field.
        if (parquetType.isPrimitive()) {
            return new RepeatedPrimitiveConverter(parquetType, updater);
        }
        else {
            return new RepeatedGroupConverter(parquetType, updater);
        }
    }
    else {
        return newConverter(parquetType, updater);
    }
}
 

Converter defaultGroupConverter(OutputMutator mutator, GroupType groupType, final String nameForChild,
                                Collection<SchemaPath> c, List<Field> arrowSchema) {

  if (groupType.getOriginalType() == OriginalType.LIST && LogicalListL1Converter.isSupportedSchema(groupType)) {
    return new LogicalListL1Converter(
      nameForChild,
      mutator,
      getWriterProvider(),
      groupType,
      c,
      options,
      arrowSchema,
      containsCorruptedDates,
      readInt96AsTimeStamp);
  }

  final MapWriter map;
  if (groupType.isRepetition(REPEATED)) {
    if (arrowSchema != null) {
      //TODO assert this should never occur at this level
      // only parquet writer that writes arrowSchema doesn't write repeated fields except
      // as part of a LOGICAL LIST, thus this scenario (repeated + arrow schema present) can
      // only happen in LogicalList converter
      arrowSchema = handleRepeatedField(arrowSchema, groupType);
    }
    map = list(nameForChild).map();
  } else {
    map = getWriterProvider().map(nameForChild);
  }
  return new StructGroupConverter(mutator, map, groupType, c, options, arrowSchema, containsCorruptedDates, readInt96AsTimeStamp);
}
 

BagConverter(GroupType parquetSchema, FieldSchema pigSchema, ParentValueContainer parent, boolean numbersDefaultToZero, boolean columnIndexAccess) throws FrontendException {
  this.parent = parent;
  if (parquetSchema.getFieldCount() != 1) {
    throw new IllegalArgumentException("bags have only one field. " + parquetSchema + " size = " + parquetSchema.getFieldCount());
  }
  Type nestedType = parquetSchema.getType(0);

  ParentValueContainer childsParent;
  FieldSchema pigField;
  if (nestedType.isPrimitive() || nestedType.getOriginalType() == OriginalType.MAP || nestedType.getOriginalType() == OriginalType.LIST) {
    // Pig bags always contain tuples
    // In that case we need to wrap the value in an extra tuple
    childsParent = new ParentValueContainer() {
      @Override
      void add(Object value) {
        buffer.add(TF.newTuple(value));
      }};
    pigField = pigSchema.schema.getField(0).schema.getField(0);
  } else {
    childsParent = new ParentValueContainer() {
      @Override
      void add(Object value) {
        buffer.add((Tuple)value);
      }};
    pigField = pigSchema.schema.getField(0);
  }
  child = newConverter(pigField, nestedType, childsParent, numbersDefaultToZero, columnIndexAccess);
}
 

private Type filterBag(GroupType bagType, FieldSchema bagFieldSchema) throws FrontendException {
  if (LOG.isDebugEnabled()) LOG.debug("filtering BAG schema:\n" + bagType + "\nwith:\n " + bagFieldSchema);
  if (bagType.getFieldCount() != 1) {
    throw new RuntimeException("not unwrapping the right type, this should be a Bag: " + bagType);
  }
  Type nested = bagType.getType(0);
  FieldSchema innerField = bagFieldSchema.schema.getField(0);
  if (nested.isPrimitive() || nested.getOriginalType() == OriginalType.MAP || nested.getOriginalType() == OriginalType.LIST) {
    // Bags always contain tuples => we skip the extra tuple that was inserted in that case.
    innerField = innerField.schema.getField(0);
  }
  return bagType.withNewFields(filter(nested, innerField));
}
 

/**
 * Will validate the structure of the list
 * @param list the Parquet List
 */
public List3Levels(GroupType list) {
  if (list.getOriginalType() != OriginalType.LIST || list.getFields().size() != 1) {
    throw new IllegalArgumentException("invalid list type: " + list);
  }
  this.list = list;
  Type repeatedField = list.getFields().get(0);
  if (repeatedField.isPrimitive() || !repeatedField.isRepetition(REPEATED) || repeatedField.asGroupType().getFields().size() != 1) {
    throw new IllegalArgumentException("invalid list type: " + list);
  }
  this.repeated = repeatedField.asGroupType();
  this.element = repeated.getFields().get(0);
}
 

OriginalType getOriginalType(ConvertedType type) {
  switch (type) {
    case UTF8:
      return OriginalType.UTF8;
    case MAP:
      return OriginalType.MAP;
    case MAP_KEY_VALUE:
      return OriginalType.MAP_KEY_VALUE;
    case LIST:
      return OriginalType.LIST;
    case ENUM:
      return OriginalType.ENUM;
    case DECIMAL:
      return OriginalType.DECIMAL;
    case DATE:
      return OriginalType.DATE;
    case TIME_MILLIS:
      return OriginalType.TIME_MILLIS;
    case TIME_MICROS:
      return OriginalType.TIME_MICROS;
    case TIMESTAMP_MILLIS:
      return OriginalType.TIMESTAMP_MILLIS;
    case TIMESTAMP_MICROS:
      return OriginalType.TIMESTAMP_MICROS;
    case INTERVAL:
      return OriginalType.INTERVAL;
    case INT_8:
      return OriginalType.INT_8;
    case INT_16:
      return OriginalType.INT_16;
    case INT_32:
      return OriginalType.INT_32;
    case INT_64:
      return OriginalType.INT_64;
    case UINT_8:
      return OriginalType.UINT_8;
    case UINT_16:
      return OriginalType.UINT_16;
    case UINT_32:
      return OriginalType.UINT_32;
    case UINT_64:
      return OriginalType.UINT_64;
    case JSON:
      return OriginalType.JSON;
    case BSON:
      return OriginalType.BSON;
    default:
      throw new RuntimeException("Unknown converted type " + type);
  }
}