Java PrimitiveConverter类代码示例

import org.apache.parquet.io.api.PrimitiveConverter; //导入依赖的package包/类
private PrimitiveConverter newConverter(int colIdx, byte vecType, PrimitiveType parquetType) {
  switch (vecType) {
    case Vec.T_BAD:
    case Vec.T_CAT:
    case Vec.T_STR:
    case Vec.T_UUID:
    case Vec.T_TIME:
      if (OriginalType.TIMESTAMP_MILLIS.equals(parquetType.getOriginalType()) || parquetType.getPrimitiveTypeName().equals(PrimitiveType.PrimitiveTypeName.INT96)) {
        return new TimestampConverter(colIdx, _writer);
      } else {
        boolean dictSupport = parquetType.getOriginalType() == OriginalType.UTF8 || parquetType.getOriginalType() == OriginalType.ENUM;
        return new StringConverter(_writer, colIdx, dictSupport);
      }
    case Vec.T_NUM:
      return new NumberConverter(colIdx, _writer);
    default:
      throw new UnsupportedOperationException("Unsupported type " + vecType);
  }
}
 

import org.apache.parquet.io.api.PrimitiveConverter; //导入依赖的package包/类
/**
 * creates a reader for triplets
 * @param path the descriptor for the corresponding column
 * @param pageReader the underlying store to read from
 */
public ColumnReaderImpl(ColumnDescriptor path, PageReader pageReader, PrimitiveConverter converter, ParsedVersion writerVersion) {
  this.path = checkNotNull(path, "path");
  this.pageReader = checkNotNull(pageReader, "pageReader");
  this.converter = checkNotNull(converter, "converter");
  this.writerVersion = writerVersion;
  DictionaryPage dictionaryPage = pageReader.readDictionaryPage();
  if (dictionaryPage != null) {
    try {
      this.dictionary = dictionaryPage.getEncoding().initDictionary(path, dictionaryPage);
      if (converter.hasDictionarySupport()) {
        converter.setDictionary(dictionary);
      }
    } catch (IOException e) {
      throw new ParquetDecodingException("could not decode the dictionary for " + path, e);
    }
  } else {
    this.dictionary = null;
  }
  this.totalValueCount = pageReader.getTotalValueCount();
  if (totalValueCount <= 0) {
    throw new ParquetDecodingException("totalValueCount '" + totalValueCount + "' <= 0");
  }
  consume();
}
 

import org.apache.parquet.io.api.PrimitiveConverter; //导入依赖的package包/类
private PrimitiveConverter getPrimitiveConverter(ColumnDescriptor path) {
  Type currentType = schema;
  Converter currentConverter = recordConverter;
  for (String fieldName : path.getPath()) {
    final GroupType groupType = currentType.asGroupType();
    int fieldIndex = groupType.getFieldIndex(fieldName);
    currentType = groupType.getType(fieldName);
    currentConverter = currentConverter.asGroupConverter().getConverter(fieldIndex);
  }
  PrimitiveConverter converter = currentConverter.asPrimitiveConverter();
  return converter;
}
 

import org.apache.parquet.io.api.PrimitiveConverter; //导入依赖的package包/类
private State(int id, PrimitiveColumnIO primitiveColumnIO, ColumnReader column, int[] nextLevel, GroupConverter[] groupConverterPath, PrimitiveConverter primitiveConverter) {
  this.id = id;
  this.primitiveColumnIO = primitiveColumnIO;
  this.maxDefinitionLevel = primitiveColumnIO.getDefinitionLevel();
  this.maxRepetitionLevel = primitiveColumnIO.getRepetitionLevel();
  this.column = column;
  this.nextLevel = nextLevel;
  this.groupConverterPath = groupConverterPath;
  this.primitiveConverter = primitiveConverter;
  this.primitive = primitiveColumnIO.getType().asPrimitiveType().getPrimitiveTypeName();
  this.fieldPath = primitiveColumnIO.getFieldPath();
  this.primitiveField = fieldPath[fieldPath.length - 1];
  this.indexFieldPath = primitiveColumnIO.getIndexFieldPath();
  this.primitiveFieldIndex = indexFieldPath[indexFieldPath.length - 1];
}
 

import org.apache.parquet.io.api.PrimitiveConverter; //导入依赖的package包/类
public static void validateValuesForPage(int rowGroupID, int pageID, DictionaryPage dictPage, DataPage page, ColumnDescriptor columnDesc, List<?> expectedValues) {
  TestStatistics.SingletonPageReader pageReader = new TestStatistics.SingletonPageReader(dictPage, page);
  PrimitiveConverter converter = getConverter(rowGroupID, pageID, columnDesc.getType(), expectedValues);
  ColumnReaderImpl column = new ColumnReaderImpl(columnDesc, pageReader, converter, null);
  for (int i = 0; i < pageReader.getTotalValueCount(); i += 1) {
    column.writeCurrentValueToConverter();
    column.consume();
  }
}
 

import org.apache.parquet.io.api.PrimitiveConverter; //导入依赖的package包/类
private void validateStatsForPage(DataPage page, DictionaryPage dict, ColumnDescriptor desc) {
  SingletonPageReader reader = new SingletonPageReader(dict, page);
  PrimitiveConverter converter = getValidatingConverter(page, desc.getType());
  Statistics<?> stats = getStatisticsFromPageHeader(page);

  assertEquals("Statistics does not use the proper comparator",
      desc.getPrimitiveType().comparator().getClass(),
      stats.comparator().getClass());

  if (stats.isEmpty()) {
    // stats are empty if num nulls = 0 and there are no non-null values
    // this happens if stats are not written (e.g., when stats are too big)
    System.err.println(String.format(
        "No stats written for page=%s col=%s",
        page, Arrays.toString(desc.getPath())));
    return;
  }

  long numNulls = 0;
  ColumnReaderImpl column = new ColumnReaderImpl(desc, reader, converter, null);
  for (int i = 0; i < reader.getTotalValueCount(); i += 1) {
    if (column.getCurrentDefinitionLevel() >= desc.getMaxDefinitionLevel()) {
      column.writeCurrentValueToConverter();
    } else {
      numNulls += 1;
    }
    column.consume();
  }

  Assert.assertEquals(numNulls, stats.getNumNulls());

  System.err.println(String.format(
      "Validated stats min=%s max=%s nulls=%d for page=%s col=%s",
      stats.minAsString(),
      stats.maxAsString(), stats.getNumNulls(), page,
      Arrays.toString(desc.getPath())));
}
 

import org.apache.parquet.io.api.PrimitiveConverter; //导入依赖的package包/类
public PrimitiveFieldHandler(PrimitiveConverter delegate, final ThriftField field, List<TProtocol> events) {
  this.delegate = delegate;
  this.events = events;
  final byte thriftType =
      field.getType().getType() == ThriftTypeID.ENUM ?
          ThriftTypeID.I32.getThriftType() : // enums are serialized as I32
          field.getType().getType().getThriftType();
  this.readFieldBegin = new ParquetProtocol("readFieldBegin()") {
    @Override
    public TField readFieldBegin() throws TException {
      return new TField(field.getName(), thriftType, field.getFieldId());
    }
  };
}
 

import org.apache.parquet.io.api.PrimitiveConverter; //导入依赖的package包/类
private void validateStatsForPage(DataPage page, DictionaryPage dict,
                                  ColumnDescriptor desc) {
  SingletonPageReader reader = new SingletonPageReader(dict, page);
  PrimitiveConverter converter = getValidatingConverter(page, desc.getType());
  Statistics stats = getStatisticsFromPageHeader(page);

  long numNulls = 0;

  ColumnReader column = COL_READER_CTOR.newInstance(desc, reader, converter, null);
  for (int i = 0; i < reader.getTotalValueCount(); i += 1) {
    if (column.getCurrentDefinitionLevel() >= desc.getMaxDefinitionLevel()) {
      column.writeCurrentValueToConverter();
    } else {
      numNulls += 1;
    }
    column.consume();
  }

  if (numNulls != stats.getNumNulls()) {
    throw new BadStatsException("Number of nulls doesn't match.");
  }

  console.debug(String.format(
      "Validated stats min=%s max=%s nulls=%d for page=%s col=%s",
      stats.minAsString(),
      stats.maxAsString(), stats.getNumNulls(), page,
      Arrays.toString(desc.getPath())));
}
 

import org.apache.parquet.io.api.PrimitiveConverter; //导入依赖的package包/类
private ColumnReaderImpl newMemColumnReader(ColumnDescriptor path, PageReader pageReader) {
  PrimitiveConverter converter = getPrimitiveConverter(path);
  return new ColumnReaderImpl(path, pageReader, converter, writerVersion);
}
 

import org.apache.parquet.io.api.PrimitiveConverter; //导入依赖的package包/类
public FilteringPrimitiveConverter(PrimitiveConverter delegate, ValueInspector[] valueInspectors) {
  this.delegate = checkNotNull(delegate, "delegate");
  this.valueInspectors = checkNotNull(valueInspectors, "valueInspectors");
}
 

import org.apache.parquet.io.api.PrimitiveConverter; //导入依赖的package包/类
@Override
public void addValueToPrimitiveConverter(
    PrimitiveConverter primitiveConverter, ColumnReader columnReader) {
  primitiveConverter.addLong(columnReader.getLong());
}
 

import org.apache.parquet.io.api.PrimitiveConverter; //导入依赖的package包/类
public PrimitiveCounter(PrimitiveConverter delegate) {
  this.delegate = delegate;
}