Java AvroParquetWriter类代码示例

import parquet.avro.AvroParquetWriter; //导入依赖的package包/类
/**
 * Write the file.
 *
 * @param args the command-line arguments
 * @return the process exit code
 * @throws Exception if something goes wrong
 */
public int run(final String[] args) throws Exception {

  Cli cli = Cli.builder().setArgs(args).addOptions(CliCommonOpts.IOFileOpts.values()).build();
  int result = cli.runCmd();

  if (result != 0) {
    return result;
  }

  File inputFile = new File(cli.getArgValueAsString(CliCommonOpts.IOFileOpts.INPUT));
  Path outputPath = new Path(cli.getArgValueAsString(CliCommonOpts.IOFileOpts.OUTPUT));

  AvroParquetWriter<Stock> writer =
      new AvroParquetWriter<Stock>(outputPath, Stock.SCHEMA$,
          CompressionCodecName.SNAPPY,
          ParquetWriter.DEFAULT_BLOCK_SIZE,
          ParquetWriter.DEFAULT_PAGE_SIZE,
          true);

  for (Stock stock : AvroStockUtils.fromCsvFile(inputFile)) {
    writer.write(stock);
  }

  writer.close();

  return 0;
}
 

import parquet.avro.AvroParquetWriter; //导入依赖的package包/类
@Override
public void open() {
  Preconditions.checkState(state.equals(ReaderWriterState.NEW),
    "Unable to open a writer from state:%s", state);

  logger.debug(
    "Opening data file with pathTmp:{} (final path will be path:{})",
    pathTmp, path);

  try {
    CompressionCodecName codecName = CompressionCodecName.UNCOMPRESSED;
    if (enableCompression) {
       if (SnappyCodec.isNativeCodeLoaded()) {
         codecName = CompressionCodecName.SNAPPY;
       } else {
         logger.warn("Compression enabled, but Snappy native code not loaded. " +
             "Parquet file will not be compressed.");
       }
    }
    avroParquetWriter = new AvroParquetWriter<E>(fileSystem.makeQualified(pathTmp),
        schema, codecName, DEFAULT_BLOCK_SIZE,
        ParquetWriter.DEFAULT_PAGE_SIZE);
  } catch (IOException e) {
    throw new DatasetWriterException("Unable to create writer to path:" + pathTmp, e);
  }

  state = ReaderWriterState.OPEN;
}
 

import parquet.avro.AvroParquetWriter; //导入依赖的package包/类
public static void main(String[] args) {

		String inputFile = null;
		String outputFile = null;
		
		HelpFormatter formatter = new HelpFormatter();
		// create Options object
		Options options = new Options();

		// add t option
		options.addOption("i", true, "input avro file");
		options.addOption("o", true, "output Parquet file");
		CommandLineParser parser = new DefaultParser();
		CommandLine cmd;
		try {
			cmd = parser.parse(options, args);
			inputFile = cmd.getOptionValue("i");
			if (inputFile == null) {
				formatter.printHelp("AvroToParquet", options);
				return;
			}
			outputFile = cmd.getOptionValue("o");
		} catch (ParseException exc) {
			System.err.println("Problem with command line parameters: " + exc.getMessage());
			return;
		}

		File avroFile = new File(inputFile);

		if (!avroFile.exists()) {
			System.err.println("Could not open file: " + inputFile);
			return;
		}
		try {

			DatumReader<GenericRecord> datumReader = new GenericDatumReader<GenericRecord>();
			DataFileReader<GenericRecord> dataFileReader;
			dataFileReader = new DataFileReader<GenericRecord>(avroFile, datumReader);
			Schema avroSchema = dataFileReader.getSchema();

			// choose compression scheme
			CompressionCodecName compressionCodecName = CompressionCodecName.SNAPPY;

			// set Parquet file block size and page size values
			int blockSize = 256 * 1024 * 1024;
			int pageSize = 64 * 1024;

			String base = FilenameUtils.removeExtension(avroFile.getAbsolutePath()) + ".parquet";
			if(outputFile != null) {
				File file = new File(outputFile);
				base = file.getAbsolutePath();
			}
			
			Path outputPath = new Path("file:///"+base);

			// the ParquetWriter object that will consume Avro GenericRecords
			ParquetWriter<GenericRecord> parquetWriter;
			parquetWriter = new AvroParquetWriter<GenericRecord>(outputPath, avroSchema, compressionCodecName, blockSize, pageSize);
			for (GenericRecord record : dataFileReader) {
				parquetWriter.write(record);
			}
			dataFileReader.close();
			parquetWriter.close();
		} catch (IOException e) {
			System.err.println("Caught exception: " + e.getMessage());
		}
	}
 

import parquet.avro.AvroParquetWriter; //导入依赖的package包/类
private int doExport(
        final IGPValue hadoopPropValue,
        final IGPValue featureClassValue,
        final IGPValue schemaValue,
        final IGPValue outputValue) throws Exception
{
    int count = 0;
    final IFeatureClass[] featureClasses = new IFeatureClass[]{new IFeatureClassProxy()};
    gpUtilities.decodeFeatureLayer(featureClassValue, featureClasses, null);
    final FeatureClass featureClass = new FeatureClass(featureClasses[0]);
    try
    {
        final int wkid = getWkid(featureClass);

        final Configuration configuration = createConfiguration(hadoopPropValue.getAsText());

        final Schema schema = parseSchema(schemaValue.getAsText(), configuration);

        final Path path = new Path(outputValue.getAsText());
        path.getFileSystem(configuration).delete(path, true);
        final AvroParquetWriter<GenericRecord> writer = new AvroParquetWriter<GenericRecord>(path, schema);
        try
        {
            final IFeatureCursor cursor = featureClass.search(null, false);
            try
            {
                final IFields fields = cursor.getFields();
                IFeature feature = cursor.nextFeature();
                try
                {
                    while (feature != null)
                    {
                        final IGeometry shape = feature.getShape();
                        if (shape instanceof Point)
                        {
                            writer.write(buildRecord(schema, wkid, fields, feature, (Point) shape));
                            count++;
                        }
                        else if (shape instanceof Polyline)
                        {
                            writer.write(buildRecord(schema, wkid, fields, feature, (Polyline) shape));
                            count++;
                        }
                        else if (shape instanceof Polygon)
                        {
                            writer.write(buildRecord(schema, wkid, fields, feature, (Polygon) shape));
                            count++;
                        }
                        feature = cursor.nextFeature();
                    }
                }
                finally
                {
                    Cleaner.release(fields);
                }
            }
            finally
            {
                Cleaner.release(cursor);
            }
        }
        finally
        {
            writer.close();
        }
    }
    finally
    {
        Cleaner.release(featureClass);
    }
    return count;
}