Java ColumnDescriptor.getMaxRepetitionLevel方法代码示例

import org.apache.parquet.column.ColumnDescriptor; //导入方法依赖的package包/类
private static void showDetails(PrettyPrintWriter out, PrimitiveType type, int depth, MessageType container, List<String> cpath) {
  String name = Strings.repeat(".", depth) + type.getName();
  OriginalType otype = type.getOriginalType();
  Repetition rep = type.getRepetition();
  PrimitiveTypeName ptype = type.getPrimitiveTypeName();

  out.format("%s: %s %s", name, rep, ptype);
  if (otype != null) out.format(" O:%s", otype);

  if (container != null) {
    cpath.add(type.getName());
    String[] paths = cpath.toArray(new String[cpath.size()]);
    cpath.remove(cpath.size() - 1);

    ColumnDescriptor desc = container.getColumnDescription(paths);

    int defl = desc.getMaxDefinitionLevel();
    int repl = desc.getMaxRepetitionLevel();
    out.format(" R:%d D:%d", repl, defl);
  }
  out.println();
}
 

import org.apache.parquet.column.ColumnDescriptor; //导入方法依赖的package包/类
/**
 * Returns data type length for a given {@see ColumnDescriptor} and it's corresponding
 * {@see SchemaElement}. Neither is enough information alone as the max
 * repetition level (indicating if it is an array type) is in the ColumnDescriptor and
 * the length of a fixed width field is stored at the schema level.
 *
 * @return the length if fixed width, else -1
 */
private int getDataTypeLength(ColumnDescriptor column, SchemaElement se) {
  if (column.getType() != PrimitiveType.PrimitiveTypeName.BINARY) {
    if (column.getMaxRepetitionLevel() > 0) {
      return -1;
    }
    if (column.getType() == PrimitiveType.PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY) {
      return se.getType_length() * 8;
    } else {
      return getTypeLengthInBits(column.getType());
    }
  } else {
    return -1;
  }
}
 

import org.apache.parquet.column.ColumnDescriptor; //导入方法依赖的package包/类
private TypeProtos.DataMode getDataMode(ColumnDescriptor column) {
  if (column.getMaxRepetitionLevel() > 0 ) {
    return DataMode.REPEATED;
  } else if (column.getMaxDefinitionLevel() == 0) {
    return TypeProtos.DataMode.REQUIRED;
  } else {
    return TypeProtos.DataMode.OPTIONAL;
  }
}
 

import org.apache.parquet.column.ColumnDescriptor; //导入方法依赖的package包/类
private static TypeProtos.DataMode getDataMode(ColumnDescriptor column) {
  if (column.getMaxRepetitionLevel() > 0 ) {
    return TypeProtos.DataMode.REPEATED;
  } else if (column.getMaxDefinitionLevel() == 0) {
    return TypeProtos.DataMode.REQUIRED;
  } else {
    return TypeProtos.DataMode.OPTIONAL;
  }
}
 

import org.apache.parquet.column.ColumnDescriptor; //导入方法依赖的package包/类
private static boolean isComplex(ParquetMetadata footer) {
  MessageType schema = footer.getFileMetaData().getSchema();

  for (Type type : schema.getFields()) {
    if (!type.isPrimitive()) {
      return true;
    }
  }
  for (ColumnDescriptor col : schema.getColumns()) {
    if (col.getMaxRepetitionLevel() > 0) {
      return true;
    }
  }
  return false;
}
 

import org.apache.parquet.column.ColumnDescriptor; //导入方法依赖的package包/类
private <T extends Comparable<T>> void validateColumn(Column<T> column) {
  ColumnPath path = column.getColumnPath();

  Class<?> alreadySeen = columnTypesEncountered.get(path);
  if (alreadySeen != null && !alreadySeen.equals(column.getColumnType())) {
    throw new IllegalArgumentException("Column: "
        + path.toDotString()
        + " was provided with different types in the same predicate."
        + " Found both: (" + alreadySeen + ", " + column.getColumnType() + ")");
  }

  if (alreadySeen == null) {
    columnTypesEncountered.put(path, column.getColumnType());
  }

  ColumnDescriptor descriptor = getColumnDescriptor(path);
  if (descriptor == null) {
    // the column is missing from the schema. evaluation uses calls
    // updateNull() a value is missing, so this will be handled correctly.
    return;
  }

  if (descriptor.getMaxRepetitionLevel() > 0) {
    throw new IllegalArgumentException("FilterPredicates do not currently support repeated columns. "
        + "Column " + path.toDotString() + " is repeated.");
  }

  ValidTypeMap.assertTypeValid(column, descriptor.getType());
}