Java Converter类代码示例

import parquet.io.api.Converter; //导入依赖的package包/类
public PrestoReadSupport(boolean useParquetColumnNames, List<HiveColumnHandle> columns, MessageType messageType)
{
    this.columns = columns;
    this.useParquetColumnNames = useParquetColumnNames;

    ImmutableList.Builder<Converter> converters = ImmutableList.builder();
    for (int i = 0; i < columns.size(); i++) {
        HiveColumnHandle column = columns.get(i);
        if (!column.isPartitionKey()) {
            parquet.schema.Type parquetType = getParquetType(column, messageType, useParquetColumnNames);
            if (parquetType == null) {
                continue;
            }
            if (parquetType.isPrimitive()) {
                converters.add(new ParquetPrimitiveColumnConverter(i));
            }
            else {
                converters.add(new ParquetColumnConverter(createGroupConverter(types[i], parquetType.getName(), parquetType, i), i));
            }
        }
    }
    this.converters = converters.build();
}
 

import parquet.io.api.Converter; //导入依赖的package包/类
public AvroUnionConverter(ParentValueContainer parent, Type parquetSchema,
                          Schema avroSchema) {
  this.parent = parent;
  GroupType parquetGroup = parquetSchema.asGroupType();
  this.memberConverters = new Converter[ parquetGroup.getFieldCount()];

  int parquetIndex = 0;
  for (int index = 0; index < avroSchema.getTypes().size(); index++) {
    Schema memberSchema = avroSchema.getTypes().get(index);
    if (!memberSchema.getType().equals(Schema.Type.NULL)) {
      Type memberType = parquetGroup.getType(parquetIndex);
      memberConverters[parquetIndex] = newConverter(memberSchema, memberType, new ParentValueContainer() {
        @Override
        void add(Object value) {
          Preconditions.checkArgument(memberValue==null, "Union is resolving to more than one type");
          memberValue = value;
        }
      });
      parquetIndex++; // Note for nulls the parquetIndex id not increased
    }
  }
}
 

import parquet.io.api.Converter; //导入依赖的package包/类
private Converter createConverter(Type field) {
  if (field.isPrimitive()) {
    OriginalType otype = field.getOriginalType();
    if (otype != null) {
      switch (otype) {
        case MAP: break;
        case LIST: break;
        case UTF8: return new StringConverter(field.getName());
        case MAP_KEY_VALUE: break;
        case ENUM: break;
      }
    }

    return new SimplePrimitiveConverter(field.getName());
  }

  return new SimpleRecordConverter(field.asGroupType(), field.getName(), this);
}
 

import parquet.io.api.Converter; //导入依赖的package包/类
@Override
public Converter getConverter(int fieldIndex)
{
    if (fieldIndex == 0) {
        return (Converter) elementConverter;
    }
    throw new IllegalArgumentException("LIST field must be 0 not " + fieldIndex);
}
 

import parquet.io.api.Converter; //导入依赖的package包/类
/**
 * Creates a new TajoRecordConverter.
 *
 * @param parquetSchema The Parquet schema of the projection.
 * @param tajoReadSchema The Tajo schema of the table.
 * @param projectionMap An array mapping the projection column to the column
 *                      index in the table.
 */
public TajoRecordConverter(GroupType parquetSchema, Schema tajoReadSchema,
                           int[] projectionMap) {
  this.parquetSchema = parquetSchema;
  this.tajoReadSchema = tajoReadSchema;
  this.projectionMap = projectionMap;
  this.tupleSize = tajoReadSchema.size();

  // The projectionMap.length does not match parquetSchema.getFieldCount()
  // when the projection contains NULL_TYPE columns. We will skip over the
  // NULL_TYPE columns when we construct the converters and populate the
  // NULL_TYPE columns with NullDatums in start().
  int index = 0;
  this.converters = new Converter[parquetSchema.getFieldCount()];
  for (int i = 0; i < projectionMap.length; ++i) {
    final int projectionIndex = projectionMap[i];
    Column column = tajoReadSchema.getColumn(projectionIndex);
    if (column.getDataType().getType() == TajoDataTypes.Type.NULL_TYPE) {
      continue;
    }
    Type type = parquetSchema.getType(index);
    converters[index] = newConverter(column, type, new ParentValueContainer() {
      @Override
      void add(Object value) {
        TajoRecordConverter.this.set(projectionIndex, value);
      }
    });
    ++index;
  }
}
 

import parquet.io.api.Converter; //导入依赖的package包/类
private Converter newConverter(Column column, Type type,
                               ParentValueContainer parent) {
  DataType dataType = column.getDataType();
  switch (dataType.getType()) {
    case BOOLEAN:
      return new FieldBooleanConverter(parent);
    case BIT:
      return new FieldBitConverter(parent);
    case CHAR:
      return new FieldCharConverter(parent);
    case INT2:
      return new FieldInt2Converter(parent);
    case INT4:
      return new FieldInt4Converter(parent);
    case INT8:
      return new FieldInt8Converter(parent);
    case FLOAT4:
      return new FieldFloat4Converter(parent);
    case FLOAT8:
      return new FieldFloat8Converter(parent);
    case INET4:
      return new FieldInet4Converter(parent);
    case INET6:
      throw new RuntimeException("No converter for INET6");
    case TEXT:
      return new FieldTextConverter(parent);
    case PROTOBUF:
      return new FieldProtobufConverter(parent, dataType);
    case BLOB:
      return new FieldBlobConverter(parent);
    case NULL_TYPE:
      throw new RuntimeException("No converter for NULL_TYPE.");
    default:
      throw new RuntimeException("Unsupported data type");
  }
}
 

import parquet.io.api.Converter; //导入依赖的package包/类
private static Converter newConverter(Schema schema, Type type,
    ParentValueContainer parent) {
  if (schema.getType().equals(Schema.Type.BOOLEAN)) {
    return new FieldBooleanConverter(parent);
  } else if (schema.getType().equals(Schema.Type.INT)) {
    return new FieldIntegerConverter(parent);
  } else if (schema.getType().equals(Schema.Type.LONG)) {
    return new FieldLongConverter(parent);
  } else if (schema.getType().equals(Schema.Type.FLOAT)) {
    return new FieldFloatConverter(parent);
  } else if (schema.getType().equals(Schema.Type.DOUBLE)) {
    return new FieldDoubleConverter(parent);
  } else if (schema.getType().equals(Schema.Type.BYTES)) {
    return new FieldBytesConverter(parent);
  } else if (schema.getType().equals(Schema.Type.STRING)) {
    return new FieldStringConverter(parent);
  } else if (schema.getType().equals(Schema.Type.RECORD)) {
    return new HMAvroConverter(parent, type.asGroupType(), schema);
  } else if (schema.getType().equals(Schema.Type.ENUM)) {
    return new FieldEnumConverter(parent,schema);
  } else if (schema.getType().equals(Schema.Type.ARRAY)) {
    return new AvroArrayConverter(parent, type, schema);
  } else if (schema.getType().equals(Schema.Type.MAP)) {
    return new MapConverter(parent, type, schema);
  } else if (schema.getType().equals(Schema.Type.UNION)) {
    return new AvroUnionConverter(parent, type, schema);
  } else if (schema.getType().equals(Schema.Type.FIXED)) {
    return new FieldFixedConverter(parent, schema);
  }
  throw new UnsupportedOperationException(String.format("Cannot convert Avro type: %s" +
      " (Parquet type: %s) ", schema, type));
}
 

import parquet.io.api.Converter; //导入依赖的package包/类
@Override
public Converter getConverter(int fieldIndex) {
  if (fieldIndex == 0) {
    return keyConverter;
  } else if (fieldIndex == 1) {
    return valueConverter;
  }
  throw new IllegalArgumentException("only the key (0) and value (1) fields expected: " + fieldIndex);
}
 

import parquet.io.api.Converter; //导入依赖的package包/类
public SimpleRecordConverter(GroupType schema, String name, SimpleRecordConverter parent) {
  this.converters = new Converter[schema.getFieldCount()];
  this.parent = parent;
  this.name = name;

  int i = 0;
  for (Type field: schema.getFields()) {
    converters[i++] = createConverter(field);
  }
}
 

import parquet.io.api.Converter; //导入依赖的package包/类
@Override
public Converter getConverter(int i) {
  return converters.get(i);
}
 

import parquet.io.api.Converter; //导入依赖的package包/类
public ParquetRecordConverter(List<Converter> converters)
{
    groupConverter = new ParquetGroupConverter(converters);
}
 

import parquet.io.api.Converter; //导入依赖的package包/类
public ParquetGroupConverter(List<Converter> converters)
{
    this.converters = converters;
}
 

import parquet.io.api.Converter; //导入依赖的package包/类
@Override
public Converter getConverter( int fieldIndex ) {
  return converters[fieldIndex];
}
 

import parquet.io.api.Converter; //导入依赖的package包/类
@Override
public Converter getConverter(int fieldIndex) {
  return converters[fieldIndex];
}
 

import parquet.io.api.Converter; //导入依赖的package包/类
/**
 * Gets the converter for a specific field.
 *
 * @param fieldIndex Index of the field in the projection.
 * @return The converter for the field.
 */
@Override
public Converter getConverter(int fieldIndex) {
  return converters[fieldIndex];
}