Java WriteSupport类代码示例

import org.apache.parquet.hadoop.api.WriteSupport; //导入依赖的package包/类
/**
 * @param parquetFileWriter the file to write to
 * @param writeSupport the class to convert incoming records
 * @param schema the schema of the records
 * @param extraMetaData extra meta data to write in the footer of the file
 * @param rowGroupSize the size of a block in the file (this will be approximate)
 * @param compressor the codec used to compress
 */
public InternalParquetRecordWriter(
    ParquetFileWriter parquetFileWriter,
    WriteSupport<T> writeSupport,
    MessageType schema,
    Map<String, String> extraMetaData,
    long rowGroupSize,
    BytesCompressor compressor,
    boolean validating,
    ParquetProperties props) {
  this.parquetFileWriter = parquetFileWriter;
  this.writeSupport = checkNotNull(writeSupport, "writeSupport");
  this.schema = schema;
  this.extraMetaData = extraMetaData;
  this.rowGroupSize = rowGroupSize;
  this.rowGroupSizeThreshold = rowGroupSize;
  this.nextRowGroupSize = rowGroupSizeThreshold;
  this.compressor = compressor;
  this.validating = validating;
  this.props = props;
  initStore();
}
 

import org.apache.parquet.hadoop.api.WriteSupport; //导入依赖的package包/类
/**
 *
 * @param w the file to write to
 * @param writeSupport the class to convert incoming records
 * @param schema the schema of the records
 * @param extraMetaData extra meta data to write in the footer of the file
 * @param blockSize the size of a block in the file (this will be approximate)
 * @param compressor the compressor used to compress the pages
 * @param dictionaryPageSize the threshold for dictionary size
 * @param enableDictionary to enable the dictionary
 * @param validating if schema validation should be turned on
 */
@Deprecated
public ParquetRecordWriter(
    ParquetFileWriter w,
    WriteSupport<T> writeSupport,
    MessageType schema,
    Map<String, String> extraMetaData,
    int blockSize, int pageSize,
    BytesCompressor compressor,
    int dictionaryPageSize,
    boolean enableDictionary,
    boolean validating,
    WriterVersion writerVersion) {
  ParquetProperties props = ParquetProperties.builder()
      .withPageSize(pageSize)
      .withDictionaryPageSize(dictionaryPageSize)
      .withDictionaryEncoding(enableDictionary)
      .withWriterVersion(writerVersion)
      .build();
  internalWriter = new InternalParquetRecordWriter<T>(w, writeSupport, schema,
      extraMetaData, blockSize, compressor, validating, props);
  this.memoryManager = null;
  this.codecFactory = null;
}
 

import org.apache.parquet.hadoop.api.WriteSupport; //导入依赖的package包/类
/**
 * Create a new ParquetWriter.
 *
 * @param file the file to create
 * @param writeSupport the implementation to write a record to a RecordConsumer
 * @param compressionCodecName the compression codec to use
 * @param blockSize the block size threshold
 * @param pageSize the page size threshold
 * @param dictionaryPageSize the page size threshold for the dictionary pages
 * @param enableDictionary to turn dictionary encoding on
 * @param validating to turn on validation using the schema
 * @param writerVersion version of parquetWriter from {@link ParquetProperties.WriterVersion}
 * @param conf Hadoop configuration to use while accessing the filesystem
 * @throws IOException
 */
@Deprecated
public ParquetWriter(
    Path file,
    WriteSupport<T> writeSupport,
    CompressionCodecName compressionCodecName,
    int blockSize,
    int pageSize,
    int dictionaryPageSize,
    boolean enableDictionary,
    boolean validating,
    WriterVersion writerVersion,
    Configuration conf) throws IOException {
  this(file, ParquetFileWriter.Mode.CREATE, writeSupport,
      compressionCodecName, blockSize, pageSize, dictionaryPageSize,
      enableDictionary, validating, writerVersion, conf);
}
 

import org.apache.parquet.hadoop.api.WriteSupport; //导入依赖的package包/类
@Override
public org.apache.parquet.hadoop.api.WriteSupport.WriteContext init(final Configuration configuration) {
    final Map<String, String> extraMeta = new HashMap<>();
    if (null != sparkSchema) {
        extraMeta.put(ParquetReadSupport.SPARK_METADATA_KEY(), sparkSchema.json());
    }
    return new WriteContext(schema, extraMeta);
}
 

import org.apache.parquet.hadoop.api.WriteSupport; //导入依赖的package包/类
@Deprecated
ParquetElementWriter(final Path file, final WriteSupport<Element> writeSupport,
                     final CompressionCodecName compressionCodecName,
                     final int blockSize, final int pageSize, final boolean enableDictionary,
                     final boolean enableValidation,
                     final ParquetProperties.WriterVersion writerVersion,
                     final Configuration conf)
        throws IOException {
    super(file, writeSupport, compressionCodecName, blockSize, pageSize,
            pageSize, enableDictionary, enableValidation, writerVersion, conf);
}
 

import org.apache.parquet.hadoop.api.WriteSupport; //导入依赖的package包/类
@Override
public org.apache.parquet.hadoop.api.WriteSupport.WriteContext init(Configuration configuration) {
  // if present, prefer the schema passed to the constructor
  if (schema == null) {
    schema = getSchema(configuration);
  }
  return new WriteContext(schema, this.extraMetaData);
}
 

import org.apache.parquet.hadoop.api.WriteSupport; //导入依赖的package包/类
public static Class<?> getWriteSupportClass(Configuration configuration) {
  final String className = configuration.get(WRITE_SUPPORT_CLASS);
  if (className == null) {
    return null;
  }
  final Class<?> writeSupportClass = ConfigurationUtil.getClassFromConfig(configuration, WRITE_SUPPORT_CLASS, WriteSupport.class);
  return writeSupportClass;
}
 

import org.apache.parquet.hadoop.api.WriteSupport; //导入依赖的package包/类
/**
 * @param conf
 * @return an appropriate WriteSupport for the object model.
 */
protected WriteSupport<Group> getWriteSupport(Configuration conf)
{
    return new GroupWriteSupport(type, extraMetaData);
}
 

import org.apache.parquet.hadoop.api.WriteSupport; //导入依赖的package包/类
@Override
protected WriteSupport<Way> getWriteSupport(Configuration conf) {
    return new WayWriteSupport(excludeMetadata);
}
 

import org.apache.parquet.hadoop.api.WriteSupport; //导入依赖的package包/类
@Override
protected WriteSupport<Element> getWriteSupport(final Configuration conf) {
    return new ElementWriteSupport(type, isEntity, converter, sparkSchema);
}
 

import org.apache.parquet.hadoop.api.WriteSupport; //导入依赖的package包/类
@Override
protected WriteSupport<PageReader> getWriteSupport(Configuration conf)
{
    return new EmbulkWriteSupport(schema, timestampFormatters);
}
 

import org.apache.parquet.hadoop.api.WriteSupport; //导入依赖的package包/类
protected WriteSupport<T> getWriteSupport(Configuration conf) {
  return new AvroWriteSupport((new AvroSchemaConverterLogicalTypesPre19(conf)).convert(this.schema), this.schema, this.model);
}
 

import org.apache.parquet.hadoop.api.WriteSupport; //导入依赖的package包/类
@Override
protected WriteSupport<Group> getWriteSupport(Configuration conf) {
  return new GroupWriteSupport(type, extraMetaData);
}
 

import org.apache.parquet.hadoop.api.WriteSupport; //导入依赖的package包/类
public RecordWriter<Void, T> getRecordWriter(Configuration conf, Path file, CompressionCodecName codec)
      throws IOException, InterruptedException {
  final WriteSupport<T> writeSupport = getWriteSupport(conf);

  ParquetProperties props = ParquetProperties.builder()
      .withPageSize(getPageSize(conf))
      .withDictionaryPageSize(getDictionaryPageSize(conf))
      .withDictionaryEncoding(getEnableDictionary(conf))
      .withWriterVersion(getWriterVersion(conf))
      .estimateRowCountForPageSizeCheck(getEstimatePageSizeCheck(conf))
      .withMinRowCountForPageSizeCheck(getMinRowCountForPageSizeCheck(conf))
      .withMaxRowCountForPageSizeCheck(getMaxRowCountForPageSizeCheck(conf))
      .build();

  long blockSize = getLongBlockSize(conf);
  int maxPaddingSize = getMaxPaddingSize(conf);
  boolean validating = getValidation(conf);

  if (LOG.isInfoEnabled()) {
    LOG.info("Parquet block size to {}", blockSize);
    LOG.info("Parquet page size to {}", props.getPageSizeThreshold());
    LOG.info("Parquet dictionary page size to {}", props.getDictionaryPageSizeThreshold());
    LOG.info("Dictionary is {}", (props.isEnableDictionary() ? "on" : "off"));
    LOG.info("Validation is {}", (validating ? "on" : "off"));
    LOG.info("Writer version is: {}", props.getWriterVersion());
    LOG.info("Maximum row group padding size is {} bytes", maxPaddingSize);
    LOG.info("Page size checking is: {}", (props.estimateNextSizeCheck() ? "estimated" : "constant"));
    LOG.info("Min row count for page size check is: {}", props.getMinRowCountForPageSizeCheck());
    LOG.info("Max row count for page size check is: {}", props.getMaxRowCountForPageSizeCheck());
  }

  WriteContext init = writeSupport.init(conf);
  ParquetFileWriter w = new ParquetFileWriter(HadoopOutputFile.fromPath(file, conf),
      init.getSchema(), Mode.CREATE, blockSize, maxPaddingSize);
  w.start();

  float maxLoad = conf.getFloat(ParquetOutputFormat.MEMORY_POOL_RATIO,
      MemoryManager.DEFAULT_MEMORY_POOL_RATIO);
  long minAllocation = conf.getLong(ParquetOutputFormat.MIN_MEMORY_ALLOCATION,
      MemoryManager.DEFAULT_MIN_MEMORY_ALLOCATION);
  synchronized (ParquetOutputFormat.class) {
    if (memoryManager == null) {
      memoryManager = new MemoryManager(maxLoad, minAllocation);
    }
  }
  if (memoryManager.getMemoryPoolRatio() != maxLoad) {
    LOG.warn("The configuration " + MEMORY_POOL_RATIO + " has been set. It should not " +
        "be reset by the new value: " + maxLoad);
  }

  return new ParquetRecordWriter<T>(
      w,
      writeSupport,
      init.getSchema(),
      init.getExtraMetaData(),
      blockSize,
      codec,
      validating,
      props,
      memoryManager,
      conf);
}
 

import org.apache.parquet.hadoop.api.WriteSupport; //导入依赖的package包/类
private static <T> WriteSupport<T> writeSupport(Schema avroSchema,
                                                GenericData model) {
  return new AvroWriteSupport<T>(
      new AvroSchemaConverter().convert(avroSchema), avroSchema, model);
}