Java OriginalType类代码示例

import parquet.schema.OriginalType; //导入依赖的package包/类
private static SchemaDescription.Field convertField( SchemaDescription schema, Type t ) {
  boolean allowNull = t.getRepetition() != Repetition.REQUIRED;
  switch ( t.asPrimitiveType().getPrimitiveTypeName() ) {
    case BINARY:
      return schema.new Field( t.getName(), t.getName(), ValueMetaInterface.TYPE_STRING, allowNull );
    case BOOLEAN:
      return schema.new Field( t.getName(), t.getName(), ValueMetaInterface.TYPE_BOOLEAN, allowNull );
    case DOUBLE:
    case FLOAT:
      return schema.new Field( t.getName(), t.getName(), ValueMetaInterface.TYPE_NUMBER, allowNull );
    case INT32:
    case INT64:
      if ( t.getOriginalType() == OriginalType.DATE || t.getOriginalType() == OriginalType.TIME_MILLIS
          || t.getOriginalType() == OriginalType.TIMESTAMP_MILLIS ) {
        return schema.new Field( t.getName(), t.getName(), ValueMetaInterface.TYPE_DATE, allowNull );
      } else {
        return schema.new Field( t.getName(), t.getName(), ValueMetaInterface.TYPE_INTEGER, allowNull );
      }
    case INT96:
      return schema.new Field( t.getName(), t.getName(), ValueMetaInterface.TYPE_DATE, allowNull );
    default:
      throw new RuntimeException( "Undefined type: " + t );
  }
}
 

import parquet.schema.OriginalType; //导入依赖的package包/类
private static void showDetails(PrettyPrintWriter out, PrimitiveType type, int depth, MessageType container, List<String> cpath) {
  String name = Strings.repeat(".", depth) + type.getName();
  OriginalType otype = type.getOriginalType();
  Repetition rep = type.getRepetition();
  PrimitiveTypeName ptype = type.getPrimitiveTypeName();

  out.format("%s: %s %s", name, rep, ptype);
  if (otype != null) out.format(" O:%s", otype);

  if (container != null) {
    cpath.add(type.getName());
    String[] paths = cpath.toArray(new String[cpath.size()]);
    cpath.remove(cpath.size() - 1);

    ColumnDescriptor desc = container.getColumnDescription(paths);

    int defl = desc.getMaxDefinitionLevel();
    int repl = desc.getMaxRepetitionLevel();
    out.format(" R:%d D:%d", repl, defl);
  }
  out.println();
}
 

import parquet.schema.OriginalType; //导入依赖的package包/类
private Converter createConverter(Type field) {
  if (field.isPrimitive()) {
    OriginalType otype = field.getOriginalType();
    if (otype != null) {
      switch (otype) {
        case MAP: break;
        case LIST: break;
        case UTF8: return new StringConverter(field.getName());
        case MAP_KEY_VALUE: break;
        case ENUM: break;
      }
    }

    return new SimplePrimitiveConverter(field.getName());
  }

  return new SimpleRecordConverter(field.asGroupType(), field.getName(), this);
}
 

import parquet.schema.OriginalType; //导入依赖的package包/类
private OriginalType getOriginalType(Type type, String[] path, int depth) {
  if (type.isPrimitive()) {
    return type.getOriginalType();
  }
  Type t = ((GroupType) type).getType(path[depth]);
  return getOriginalType(t, path, depth + 1);
}
 

import parquet.schema.OriginalType; //导入依赖的package包/类
public ColumnMetadata(SchemaPath name, PrimitiveTypeName primitiveType, OriginalType originalType,
                      Object max, Object min, Long nulls) {
  this.name = name;
  this.primitiveType = primitiveType;
  this.originalType = originalType;
  this.max = max;
  this.min = min;
  this.nulls = nulls;
}
 

import parquet.schema.OriginalType; //导入依赖的package包/类
private PrimitiveType getPrimitiveType(MaterializedField field) {
  MinorType minorType = field.getType().getMinorType();
  String name = field.getLastName();
  PrimitiveTypeName primitiveTypeName = ParquetTypeHelper.getPrimitiveTypeNameForMinorType(minorType);
  Repetition repetition = ParquetTypeHelper.getRepetitionForDataMode(field.getDataMode());
  OriginalType originalType = ParquetTypeHelper.getOriginalTypeForMinorType(minorType);
  DecimalMetadata decimalMetadata = ParquetTypeHelper.getDecimalMetadataForField(field);
  int length = ParquetTypeHelper.getLengthForMinorType(minorType);
  return new PrimitiveType(repetition, primitiveTypeName, length, name, originalType, decimalMetadata, null);
}
 

import parquet.schema.OriginalType; //导入依赖的package包/类
@Test
public void testSchemaConverterDecimal() {
    ParquetMetadataConverter converter = new ParquetMetadataConverter();
    List<SchemaElement> schemaElements = converter.toParquetSchema(
            Types.buildMessage()
                    .required(PrimitiveTypeName.BINARY)
                    .as(OriginalType.DECIMAL).precision(9).scale(2)
                    .named("aBinaryDecimal")
                    .optional(PrimitiveTypeName.FIXED_LEN_BYTE_ARRAY).length(4)
                    .as(OriginalType.DECIMAL).precision(9).scale(2)
                    .named("aFixedDecimal")
                    .named("Message")
    );
    List<SchemaElement> expected = Lists.newArrayList(
            new SchemaElement("Message").setNum_children(2),
            new SchemaElement("aBinaryDecimal")
                    .setRepetition_type(FieldRepetitionType.REQUIRED)
                    .setType(Type.BYTE_ARRAY)
                    .setConverted_type(ConvertedType.DECIMAL)
                    .setPrecision(9).setScale(2),
            new SchemaElement("aFixedDecimal")
                    .setRepetition_type(FieldRepetitionType.OPTIONAL)
                    .setType(Type.FIXED_LEN_BYTE_ARRAY)
                    .setType_length(4)
                    .setConverted_type(ConvertedType.DECIMAL)
                    .setPrecision(9).setScale(2)
    );
    Assert.assertEquals(expected, schemaElements);
}
 

import parquet.schema.OriginalType; //导入依赖的package包/类
private Type convertColumn(Column column) {
  TajoDataTypes.Type type = column.getDataType().getType();
  switch (type) {
    case BOOLEAN:
      return primitive(column.getSimpleName(),
                       PrimitiveType.PrimitiveTypeName.BOOLEAN);
    case BIT:
    case INT2:
    case INT4:
      return primitive(column.getSimpleName(),
                       PrimitiveType.PrimitiveTypeName.INT32);
    case INT8:
      return primitive(column.getSimpleName(),
                       PrimitiveType.PrimitiveTypeName.INT64);
    case FLOAT4:
      return primitive(column.getSimpleName(),
                       PrimitiveType.PrimitiveTypeName.FLOAT);
    case FLOAT8:
      return primitive(column.getSimpleName(),
                       PrimitiveType.PrimitiveTypeName.DOUBLE);
    case CHAR:
    case TEXT:
      return primitive(column.getSimpleName(),
                       PrimitiveType.PrimitiveTypeName.BINARY,
                       OriginalType.UTF8);
    case PROTOBUF:
      return primitive(column.getSimpleName(),
                       PrimitiveType.PrimitiveTypeName.BINARY);
    case BLOB:
      return primitive(column.getSimpleName(),
                       PrimitiveType.PrimitiveTypeName.BINARY);
    case INET4:
    case INET6:
      return primitive(column.getSimpleName(),
                       PrimitiveType.PrimitiveTypeName.BINARY);
    default:
      throw new RuntimeException("Cannot convert Tajo type: " + type);
  }
}
 

import parquet.schema.OriginalType; //导入依赖的package包/类
private MajorType getType(PrimitiveTypeName type, OriginalType originalType) {
  if (originalType != null) {
    switch (originalType) {
    case DECIMAL:
      return Types.optional(MinorType.DECIMAL18);
    case DATE:
      return Types.optional(MinorType.DATE);
    case TIME_MILLIS:
      return Types.optional(MinorType.TIME);
    case TIMESTAMP_MILLIS:
      return Types.optional(MinorType.TIMESTAMP);
    case UTF8:
      return Types.optional(MinorType.VARCHAR);
    case UINT_8:
      return Types.optional(MinorType.UINT1);
    case UINT_16:
      return Types.optional(MinorType.UINT2);
    case UINT_32:
      return Types.optional(MinorType.UINT4);
    case UINT_64:
      return Types.optional(MinorType.UINT8);
    case INT_8:
      return Types.optional(MinorType.TINYINT);
    case INT_16:
      return Types.optional(MinorType.SMALLINT);
    }
  }

  switch (type) {
  case BOOLEAN:
    return Types.optional(MinorType.BIT);
  case INT32:
    return Types.optional(MinorType.INT);
  case INT64:
    return Types.optional(MinorType.BIGINT);
  case FLOAT:
    return Types.optional(MinorType.FLOAT4);
  case DOUBLE:
    return Types.optional(MinorType.FLOAT8);
  case BINARY:
  case FIXED_LEN_BYTE_ARRAY:
  case INT96:
    return Types.optional(MinorType.VARBINARY);
  default:
    // Should never hit this
    throw new UnsupportedOperationException("Unsupported type:" + type);
  }
}
 

import parquet.schema.OriginalType; //导入依赖的package包/类
public static OriginalType getOriginalTypeForMinorType(MinorType minorType) {
  return originalTypeMap.get(minorType);
}
 

import parquet.schema.OriginalType; //导入依赖的package包/类
@Test
public void testExportParquetString() throws Exception {

    List<Integer> dynamicCols = new ArrayList<>();
    List<Type> schemaTypes = new ArrayList<>();
    schemaTypes.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveType.PrimitiveTypeName.BINARY, "c1", OriginalType.UTF8));
    schemaTypes.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveType.PrimitiveTypeName.BINARY, "c2", OriginalType.UTF8));
    schemaTypes.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveType.PrimitiveTypeName.BINARY, "v1", OriginalType.UTF8));
    schemaTypes.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveType.PrimitiveTypeName.BINARY, "v2", OriginalType.UTF8));
    schemaTypes.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveType.PrimitiveTypeName.BINARY, "s1", OriginalType.UTF8));
    schemaTypes.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveType.PrimitiveTypeName.BINARY, "charnull", OriginalType.UTF8));
    schemaTypes.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveType.PrimitiveTypeName.BINARY, "varcharnull", OriginalType.UTF8));
    schemaTypes.add(new PrimitiveType(Type.Repetition.OPTIONAL, PrimitiveType.PrimitiveTypeName.BINARY, "stringnull", OriginalType.UTF8));

    List<List<Object>> dataSet = new ArrayList<>();
    List<Object> row = new ArrayList<>();
    row.add("a");
    row.add("aaaaaaaaaa");
    row.add("b");
    row.add("bbbbbbbbbb");
    row.add("cccccccccccccccccccc");
    row.add(null);
    row.add(null);
    row.add(null);
    addRow(dataSet, row);
    ExaIterator iter = new ExaIteratorDummy(dataSet);

    File tempFile = new File(testFolder.getRoot(),UUID.randomUUID().toString().replaceAll("-", "") + ".parq");

    HdfsSerDeExportService.exportToParquetTable(testFolder.getRoot().toString(), "hdfs", false, null, tempFile.getName(), null, "uncompressed", schemaTypes, FIRST_DATA_COLUMN, dynamicCols, iter);

    ExaIterator ctx = mock(ExaIterator.class);
    List<HCatTableColumn> columns = new ArrayList<>();
    columns.add(new HCatTableColumn("c1", "char(1)"));
    columns.add(new HCatTableColumn("c2", "char(12)"));
    columns.add(new HCatTableColumn("v1", "varchar(1)"));
    columns.add(new HCatTableColumn("v2", "varchar(10)"));
    columns.add(new HCatTableColumn("s1", "string"));
    columns.add(new HCatTableColumn("charnull", "char(1)"));
    columns.add(new HCatTableColumn("varcharnulll", "varchar(1)"));
    columns.add(new HCatTableColumn("stringnull", "string"));

    List<HCatTableColumn> partitionColumns = null;
    importFile(ctx, columns, partitionColumns, tempFile.getCanonicalPath(), PARQUET_INPUT_FORMAT_CLASS_NAME, PARQUET_SERDE_CLASS_NAME);

    int expectedNumRows = 1;
    verify(ctx, times(expectedNumRows)).emit(anyVararg());
    verify(ctx, times(1)).emit(
            eq("a"),
            eq("aaaaaaaaaa  "),
            eq("b"),
            eq("bbbbbbbbbb"),
            eq("cccccccccccccccccccc"),
            eq(null),
            eq(null),
            eq(null)
    );
}
 

import parquet.schema.OriginalType; //导入依赖的package包/类
OriginalType getOriginalType(ConvertedType type) {
    switch (type) {
        case UTF8:
            return OriginalType.UTF8;
        case MAP:
            return OriginalType.MAP;
        case MAP_KEY_VALUE:
            return OriginalType.MAP_KEY_VALUE;
        case LIST:
            return OriginalType.LIST;
        case ENUM:
            return OriginalType.ENUM;
        case DECIMAL:
            return OriginalType.DECIMAL;
        case DATE:
            return OriginalType.DATE;
        case TIME_MILLIS:
            return OriginalType.TIME_MILLIS;
        case TIMESTAMP_MILLIS:
            return OriginalType.TIMESTAMP_MILLIS;
        case INTERVAL:
            return OriginalType.INTERVAL;
        case INT_8:
            return OriginalType.INT_8;
        case INT_16:
            return OriginalType.INT_16;
        case INT_32:
            return OriginalType.INT_32;
        case INT_64:
            return OriginalType.INT_64;
        case UINT_8:
            return OriginalType.UINT_8;
        case UINT_16:
            return OriginalType.UINT_16;
        case UINT_32:
            return OriginalType.UINT_32;
        case UINT_64:
            return OriginalType.UINT_64;
        case JSON:
            return OriginalType.JSON;
        case BSON:
            return OriginalType.BSON;
        default:
            throw new RuntimeException("Unknown converted type " + type);
    }
}
 

import parquet.schema.OriginalType; //导入依赖的package包/类
ConvertedType getConvertedType(OriginalType type) {
    switch (type) {
        case UTF8:
            return ConvertedType.UTF8;
        case MAP:
            return ConvertedType.MAP;
        case MAP_KEY_VALUE:
            return ConvertedType.MAP_KEY_VALUE;
        case LIST:
            return ConvertedType.LIST;
        case ENUM:
            return ConvertedType.ENUM;
        case DECIMAL:
            return ConvertedType.DECIMAL;
        case DATE:
            return ConvertedType.DATE;
        case TIME_MILLIS:
            return ConvertedType.TIME_MILLIS;
        case TIMESTAMP_MILLIS:
            return ConvertedType.TIMESTAMP_MILLIS;
        case INTERVAL:
            return ConvertedType.INTERVAL;
        case INT_8:
            return ConvertedType.INT_8;
        case INT_16:
            return ConvertedType.INT_16;
        case INT_32:
            return ConvertedType.INT_32;
        case INT_64:
            return ConvertedType.INT_64;
        case UINT_8:
            return ConvertedType.UINT_8;
        case UINT_16:
            return ConvertedType.UINT_16;
        case UINT_32:
            return ConvertedType.UINT_32;
        case UINT_64:
            return ConvertedType.UINT_64;
        case JSON:
            return ConvertedType.JSON;
        case BSON:
            return ConvertedType.BSON;
        default:
            throw new RuntimeException("Unknown original type " + type);
    }
}
 

import parquet.schema.OriginalType; //导入依赖的package包/类
private static OriginalType getOriginalType(ConvertedType type)
{
    switch (type) {
        case UTF8:
            return OriginalType.UTF8;
        case MAP:
            return OriginalType.MAP;
        case MAP_KEY_VALUE:
            return OriginalType.MAP_KEY_VALUE;
        case LIST:
            return OriginalType.LIST;
        case ENUM:
            return OriginalType.ENUM;
        case DECIMAL:
            return OriginalType.DECIMAL;
        case DATE:
            return OriginalType.DATE;
        case TIME_MILLIS:
            return OriginalType.TIME_MILLIS;
        case TIMESTAMP_MILLIS:
            return OriginalType.TIMESTAMP_MILLIS;
        case INTERVAL:
            return OriginalType.INTERVAL;
        case INT_8:
            return OriginalType.INT_8;
        case INT_16:
            return OriginalType.INT_16;
        case INT_32:
            return OriginalType.INT_32;
        case INT_64:
            return OriginalType.INT_64;
        case UINT_8:
            return OriginalType.UINT_8;
        case UINT_16:
            return OriginalType.UINT_16;
        case UINT_32:
            return OriginalType.UINT_32;
        case UINT_64:
            return OriginalType.UINT_64;
        case JSON:
            return OriginalType.JSON;
        case BSON:
            return OriginalType.BSON;
        default:
            throw new IllegalArgumentException("Unknown converted type " + type);
    }
}