本文整理汇总了Java中org.apache.parquet.format.converter.ParquetMetadataConverter.MetadataFilter方法的典型用法代码示例。如果您正苦于以下问题：Java ParquetMetadataConverter.MetadataFilter方法的具体用法？Java ParquetMetadataConverter.MetadataFilter怎么用？Java ParquetMetadataConverter.MetadataFilter使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.parquet.format.converter.ParquetMetadataConverter的用法示例。
在下文中一共展示了ParquetMetadataConverter.MetadataFilter方法的5个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: ParquetReadOptions
import org.apache.parquet.format.converter.ParquetMetadataConverter; //导入方法依赖的package包/类
ParquetReadOptions(boolean useSignedStringMinMax,
boolean useStatsFilter,
boolean useDictionaryFilter,
boolean useRecordFilter,
FilterCompat.Filter recordFilter,
ParquetMetadataConverter.MetadataFilter metadataFilter,
CompressionCodecFactory codecFactory,
ByteBufferAllocator allocator,
Map<String, String> properties) {
this.useSignedStringMinMax = useSignedStringMinMax;
this.useStatsFilter = useStatsFilter;
this.useDictionaryFilter = useDictionaryFilter;
this.useRecordFilter = useRecordFilter;
this.recordFilter = recordFilter;
this.metadataFilter = metadataFilter;
this.codecFactory = codecFactory;
this.allocator = allocator;
this.properties = Collections.unmodifiableMap(properties);
}
示例2: readFooter
import org.apache.parquet.format.converter.ParquetMetadataConverter; //导入方法依赖的package包/类
public static ParquetMetadata readFooter(final FileSystem fs, final Path file, ParquetMetadataConverter.MetadataFilter filter) throws IOException {
return readFooter(fs, fs.getFileStatus(file), filter);
}
示例3: parseChunk
import org.apache.parquet.format.converter.ParquetMetadataConverter; //导入方法依赖的package包/类
@Override
protected final ParseWriter parseChunk(int cidx, ParseReader din, ParseWriter dout) {
if (! (din instanceof FVecParseReader)) {
// TODO: Should we modify the interface to expose the underlying chunk for non-streaming parsers?
throw new IllegalStateException("We only accept parser readers backed by a Vec (no streaming support!).");
}
Chunk chunk = ((FVecParseReader) din).getChunk();
Vec vec = chunk.vec();
// extract metadata, we want to read only the row groups that have centers in this chunk
ParquetMetadataConverter.MetadataFilter chunkFilter = ParquetMetadataConverter.range(
chunk.start(), chunk.start() + chunk.len());
ParquetMetadata metadata = VecParquetReader.readFooter(_metadata, chunkFilter);
if (metadata.getBlocks().isEmpty()) {
Log.trace("Chunk #", cidx, " doesn't contain any Parquet block center.");
return dout;
}
Log.info("Processing ", metadata.getBlocks().size(), " blocks of chunk #", cidx);
VecParquetReader reader = new VecParquetReader(vec, metadata, dout, _setup.getColumnTypes());
try {
Integer recordNumber;
do {
recordNumber = reader.read();
} while (recordNumber != null);
} catch (IOException e) {
throw new RuntimeException("Failed to parse records", e);
}
long skipped = reader.getInvalidRecordCount();
if (skipped > 0)
dout.addError(new ParseWriter.ParseErr("Some records were corrupt (skipped " + skipped + ").", cidx, -1, chunk.start()));
return dout;
}
示例4: getMetadataFilter
import org.apache.parquet.format.converter.ParquetMetadataConverter; //导入方法依赖的package包/类
public ParquetMetadataConverter.MetadataFilter getMetadataFilter() {
return metadataFilter;
}
示例5: withMetadataFilter
import org.apache.parquet.format.converter.ParquetMetadataConverter; //导入方法依赖的package包/类
public Builder withMetadataFilter(ParquetMetadataConverter.MetadataFilter metadataFilter) {
this.metadataFilter = metadataFilter;
return this;
}