本文整理汇总了Java中parquet.column.page.DataPageV2类的典型用法代码示例。如果您正苦于以下问题：Java DataPageV2类的具体用法？Java DataPageV2怎么用？Java DataPageV2使用的例子？那么, 这里精选的类代码示例或许可以为您提供帮助。
DataPageV2类属于parquet.column.page包，在下文中一共展示了DataPageV2类的5个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: readDataPageV2
import parquet.column.page.DataPageV2; //导入依赖的package包/类
private long readDataPageV2(PageHeader pageHeader,
int uncompressedPageSize,
int compressedPageSize,
List<DataPage> pages)
throws IOException
{
DataPageHeaderV2 dataHeaderV2 = pageHeader.getData_page_header_v2();
int dataSize = compressedPageSize - dataHeaderV2.getRepetition_levels_byte_length() - dataHeaderV2.getDefinition_levels_byte_length();
pages.add(new DataPageV2(dataHeaderV2.getNum_rows(),
dataHeaderV2.getNum_nulls(),
dataHeaderV2.getNum_values(),
getBytesInput(dataHeaderV2.getRepetition_levels_byte_length()),
getBytesInput(dataHeaderV2.getDefinition_levels_byte_length()),
Encoding.valueOf(dataHeaderV2.getEncoding().name()),
getBytesInput(dataSize),
uncompressedPageSize,
ParquetMetadataReader.readStats(dataHeaderV2.getStatistics(),
descriptor.getColumnDescriptor().getType()),
dataHeaderV2.isIs_compressed()));
return dataHeaderV2.getNum_values();
}
示例2: readPage
import parquet.column.page.DataPageV2; //导入依赖的package包/类
@Override
public DataPage readPage()
{
if (compressedPages.isEmpty()) {
return null;
}
DataPage compressedPage = compressedPages.remove(0);
try {
if (compressedPage instanceof DataPageV1) {
DataPageV1 dataPageV1 = (DataPageV1) compressedPage;
return new DataPageV1(decompressor.decompress(dataPageV1.getBytes(),
dataPageV1.getUncompressedSize()),
dataPageV1.getValueCount(),
dataPageV1.getUncompressedSize(),
dataPageV1.getStatistics(),
dataPageV1.getRlEncoding(),
dataPageV1.getDlEncoding(),
dataPageV1.getValueEncoding());
}
else {
DataPageV2 dataPageV2 = (DataPageV2) compressedPage;
if (!dataPageV2.isCompressed()) {
return dataPageV2;
}
int uncompressedSize = Ints.checkedCast(dataPageV2.getUncompressedSize() - dataPageV2.getDefinitionLevels().size() - dataPageV2.getRepetitionLevels().size());
return DataPageV2.uncompressed(dataPageV2.getRowCount(),
dataPageV2.getNullCount(),
dataPageV2.getValueCount(),
dataPageV2.getRepetitionLevels(),
dataPageV2.getDefinitionLevels(),
dataPageV2.getDataEncoding(),
decompressor.decompress(dataPageV2.getData(), uncompressedSize),
dataPageV2.getStatistics());
}
}
catch (IOException e) {
throw new RuntimeException("Could not decompress page", e);
}
}
示例3: readBlock
import parquet.column.page.DataPageV2; //导入依赖的package包/类
public Block readBlock(int vectorSize, Type type)
{
checkArgument(currentValueCount <= totalValueCount, "Already read all values in this column chunk");
int valueCount = 0;
ParquetBlockBuilder blockBuilder = ParquetBlockBuilder.createBlockBuilder(vectorSize, columnDescriptor);
while (valueCount < vectorSize) {
if (this.page == null) {
this.page = pageReader.readPage();
if (this.page == null) {
break;
}
remainingValueCountInPage = page.getValueCount();
if (page instanceof DataPageV1) {
valuesReader = readPageV1((DataPageV1) page);
}
else {
valuesReader = readPageV2((DataPageV2) page);
}
}
int valueNumber = Math.min(this.remainingValueCountInPage, vectorSize - valueCount);
blockBuilder.readValues(this.valuesReader, valueNumber, definitionReader);
valueCount = valueCount + valueNumber;
if (valueNumber == remainingValueCountInPage) {
page = null;
valuesReader = null;
}
remainingValueCountInPage = remainingValueCountInPage - valueNumber;
currentValueCount += valueNumber;
}
return blockBuilder.buildBlock();
}
示例4: readPageV2
import parquet.column.page.DataPageV2; //导入依赖的package包/类
private ValuesReader readPageV2(DataPageV2 page)
{
repetitionReader = buildLevelRLEReader(columnDescriptor.getMaxRepetitionLevel(), page.getRepetitionLevels());
definitionReader = buildLevelRLEReader(columnDescriptor.getMaxDefinitionLevel(), page.getDefinitionLevels());
try {
return initDataReader(page.getDataEncoding(), page.getData().toByteArray(), 0, page.getValueCount());
}
catch (IOException e) {
throw new ParquetDecodingException("could not read page " + page + " in col " + columnDescriptor, e);
}
}
示例5: test
import parquet.column.page.DataPageV2; //导入依赖的package包/类
@Test
public void test() throws Exception {
Path file = new Path("target/test/TestColumnChunkPageWriteStore/test.parquet");
Path root = file.getParent();
FileSystem fs = file.getFileSystem(conf);
if (fs.exists(root)) {
fs.delete(root, true);
}
fs.mkdirs(root);
MessageType schema = MessageTypeParser.parseMessageType("message test { repeated binary bar; }");
ColumnDescriptor col = schema.getColumns().get(0);
Encoding dataEncoding = PLAIN;
int valueCount = 10;
int d = 1;
int r = 2;
int v = 3;
BytesInput definitionLevels = BytesInput.fromInt(d);
BytesInput repetitionLevels = BytesInput.fromInt(r);
Statistics<?> statistics = new BinaryStatistics();
BytesInput data = BytesInput.fromInt(v);
int rowCount = 5;
int nullCount = 1;
{
ParquetFileWriter writer = new ParquetFileWriter(conf, schema, file);
writer.start();
writer.startBlock(rowCount);
{
ColumnChunkPageWriteStore store = new ColumnChunkPageWriteStore(compressor(GZIP), schema, initialSize);
PageWriter pageWriter = store.getPageWriter(col);
pageWriter.writePageV2(
rowCount, nullCount, valueCount,
repetitionLevels, definitionLevels,
dataEncoding, data,
statistics);
store.flushToFileWriter(writer);
}
writer.endBlock();
writer.end(new HashMap<String, String>());
}
{
ParquetMetadata footer = ParquetFileReader.readFooter(conf, file, NO_FILTER);
ParquetFileReader reader = new ParquetFileReader(conf, file, footer.getBlocks(), schema.getColumns());
PageReadStore rowGroup = reader.readNextRowGroup();
PageReader pageReader = rowGroup.getPageReader(col);
DataPageV2 page = (DataPageV2) pageReader.readPage();
assertEquals(rowCount, page.getRowCount());
assertEquals(nullCount, page.getNullCount());
assertEquals(valueCount, page.getValueCount());
assertEquals(d, intValue(page.getDefinitionLevels()));
assertEquals(r, intValue(page.getRepetitionLevels()));
assertEquals(dataEncoding, page.getDataEncoding());
assertEquals(v, intValue(page.getData()));
assertEquals(statistics.toString(), page.getStatistics().toString());
reader.close();
}
}