当前位置: 首页>>代码示例>>Java>>正文


Java RecordWriter.write方法代码示例

本文整理汇总了Java中org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter.write方法的典型用法代码示例。如果您正苦于以下问题:Java RecordWriter.write方法的具体用法?Java RecordWriter.write怎么用?Java RecordWriter.write使用的例子?那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter的用法示例。


在下文中一共展示了RecordWriter.write方法的4个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: createEmptyFile

import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; //导入方法依赖的package包/类
@SuppressWarnings({"rawtypes", "unchecked"})
private static Path createEmptyFile(Path hiveScratchDir,
    HiveOutputFormat outFileFormat, JobConf job,
    int sequenceNumber, Properties props, boolean dummyRow)
        throws IOException, InstantiationException, IllegalAccessException {

  // create a dummy empty file in a new directory
  String newDir = hiveScratchDir + Path.SEPARATOR + sequenceNumber;
  Path newPath = new Path(newDir);
  FileSystem fs = newPath.getFileSystem(job);
  fs.mkdirs(newPath);
  //Qualify the path against the file system. The user configured path might contain default port which is skipped
  //in the file status. This makes sure that all paths which goes into PathToPartitionInfo are always listed status
  //file path.
  newPath = fs.makeQualified(newPath);
  String newFile = newDir + Path.SEPARATOR + "emptyFile";
  Path newFilePath = new Path(newFile);

  RecordWriter recWriter = outFileFormat.getHiveRecordWriter(job, newFilePath,
      Text.class, false, props, null);
  if (dummyRow) {
    // empty files are omitted at CombineHiveInputFormat.
    // for meta-data only query, it effectively makes partition columns disappear..
    // this could be fixed by other methods, but this seemed to be the most easy (HIVEV-2955)
    recWriter.write(new Text("empty"));  // written via HiveIgnoreKeyTextOutputFormat
  }
  recWriter.close(false);

  return newPath;
}
 
开发者ID:mini666,项目名称:hive-phoenix-handler,代码行数:31,代码来源:Utilities.java

示例2: createTestFile

import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; //导入方法依赖的package包/类
public static FileSplit createTestFile(String filePath,
        HiveOutputFormat<?, ?> outputFormat,
        @SuppressWarnings("deprecation") SerDe serDe,
        String compressionCodec,
        List<TestColumn> testColumns,
        int numRows)
        throws Exception
{
    // filter out partition keys, which are not written to the file
    testColumns = ImmutableList.copyOf(filter(testColumns, not(TestColumn::isPartitionKey)));

    JobConf jobConf = new JobConf();
    ReaderWriterProfiler.setProfilerOptions(jobConf);

    Properties tableProperties = new Properties();
    tableProperties.setProperty("columns", Joiner.on(',').join(transform(testColumns, TestColumn::getName)));
    tableProperties.setProperty("columns.types", Joiner.on(',').join(transform(testColumns, TestColumn::getType)));
    serDe.initialize(CONFIGURATION, tableProperties);

    if (compressionCodec != null) {
        CompressionCodec codec = new CompressionCodecFactory(CONFIGURATION).getCodecByName(compressionCodec);
        jobConf.set(COMPRESS_CODEC, codec.getClass().getName());
        jobConf.set(COMPRESS_TYPE, SequenceFile.CompressionType.BLOCK.toString());
    }

    RecordWriter recordWriter = createRecordWriter(new Path(filePath), CONFIGURATION);

    try {
        SettableStructObjectInspector objectInspector = getStandardStructObjectInspector(
                ImmutableList.copyOf(transform(testColumns, TestColumn::getName)),
                ImmutableList.copyOf(transform(testColumns, TestColumn::getObjectInspector)));

        Object row = objectInspector.create();

        List<StructField> fields = ImmutableList.copyOf(objectInspector.getAllStructFieldRefs());

        for (int rowNumber = 0; rowNumber < numRows; rowNumber++) {
            for (int i = 0; i < testColumns.size(); i++) {
                Object writeValue = testColumns.get(i).getWriteValue();
                if (writeValue instanceof Slice) {
                    writeValue = ((Slice) writeValue).getBytes();
                }
                objectInspector.setStructFieldData(row, fields.get(i), writeValue);
            }

            Writable record = serDe.serialize(row, objectInspector);
            recordWriter.write(record);
            if (rowNumber % STRIPE_ROWS == STRIPE_ROWS - 1) {
                flushStripe(recordWriter);
            }
        }
    }
    finally {
        recordWriter.close(false);
    }

    Path path = new Path(filePath);
    path.getFileSystem(CONFIGURATION).setVerifyChecksum(true);
    File file = new File(filePath);
    return new FileSplit(path, 0, file.length(), new String[0]);
}
 
开发者ID:y-lan,项目名称:presto,代码行数:62,代码来源:TestOrcPageSourceMemoryTracking.java

示例3: writeLineItems

import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; //导入方法依赖的package包/类
public static DataSize writeLineItems(
        File outputFile,
        HiveOutputFormat<?, ?> outputFormat,
        @SuppressWarnings("deprecation") Serializer serializer,
        CompressionType compressionType,
        List<? extends TpchColumn<?>> columns)
        throws Exception
{
    RecordWriter recordWriter = createRecordWriter(columns, outputFile, outputFormat, compressionType);

    SettableStructObjectInspector objectInspector = getStandardStructObjectInspector(transform(columns, input -> input.getColumnName()), transform(columns, input -> getObjectInspector(input)));

    Object row = objectInspector.create();

    List<StructField> fields = ImmutableList.copyOf(objectInspector.getAllStructFieldRefs());

    for (LineItem lineItem : new LineItemGenerator(1, 1, 1)) {
        objectInspector.setStructFieldData(row, fields.get(0), lineItem.getOrderKey());
        objectInspector.setStructFieldData(row, fields.get(1), lineItem.getPartKey());
        objectInspector.setStructFieldData(row, fields.get(2), lineItem.getSupplierKey());
        objectInspector.setStructFieldData(row, fields.get(3), lineItem.getLineNumber());
        objectInspector.setStructFieldData(row, fields.get(4), lineItem.getQuantity());
        objectInspector.setStructFieldData(row, fields.get(5), lineItem.getExtendedPrice());
        objectInspector.setStructFieldData(row, fields.get(6), lineItem.getDiscount());
        objectInspector.setStructFieldData(row, fields.get(7), lineItem.getTax());
        objectInspector.setStructFieldData(row, fields.get(8), lineItem.getReturnFlag());
        objectInspector.setStructFieldData(row, fields.get(9), lineItem.getStatus());
        objectInspector.setStructFieldData(row, fields.get(10), lineItem.getShipDate());
        objectInspector.setStructFieldData(row, fields.get(11), lineItem.getCommitDate());
        objectInspector.setStructFieldData(row, fields.get(12), lineItem.getReceiptDate());
        objectInspector.setStructFieldData(row, fields.get(13), lineItem.getShipInstructions());
        objectInspector.setStructFieldData(row, fields.get(14), lineItem.getShipMode());
        objectInspector.setStructFieldData(row, fields.get(15), lineItem.getComment());

        Writable record = serializer.serialize(row, objectInspector);
        recordWriter.write(record);
    }

    recordWriter.close(false);
    return getFileSize(outputFile);
}
 
开发者ID:y-lan,项目名称:presto,代码行数:42,代码来源:BenchmarkHiveFileFormats.java

示例4: createTestFile

import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; //导入方法依赖的package包/类
public static FileSplit createTestFile(String filePath,
        HiveOutputFormat<?, ?> outputFormat,
        @SuppressWarnings("deprecation") SerDe serDe,
        String compressionCodec,
        List<TestColumn> testColumns,
        int numRows)
        throws Exception
{
    // filter out partition keys, which are not written to the file
    testColumns = ImmutableList.copyOf(filter(testColumns, not(TestColumn::isPartitionKey)));

    JobConf jobConf = new JobConf();
    ReaderWriterProfiler.setProfilerOptions(jobConf);

    Properties tableProperties = new Properties();
    tableProperties.setProperty("columns", Joiner.on(',').join(transform(testColumns, TestColumn::getName)));
    tableProperties.setProperty("columns.types", Joiner.on(',').join(transform(testColumns, TestColumn::getType)));
    serDe.initialize(new Configuration(), tableProperties);

    if (compressionCodec != null) {
        CompressionCodec codec = new CompressionCodecFactory(new Configuration()).getCodecByName(compressionCodec);
        jobConf.set(COMPRESS_CODEC, codec.getClass().getName());
        jobConf.set(COMPRESS_TYPE, SequenceFile.CompressionType.BLOCK.toString());
        jobConf.set("parquet.compression", compressionCodec);
        jobConf.set("parquet.enable.dictionary", "true");
    }

    RecordWriter recordWriter = outputFormat.getHiveRecordWriter(
            jobConf,
            new Path(filePath),
            Text.class,
            compressionCodec != null,
            tableProperties,
            new Progressable()
            {
                @Override
                public void progress()
                {
                }
            }
    );

    try {
        serDe.initialize(new Configuration(), tableProperties);

        SettableStructObjectInspector objectInspector = getStandardStructObjectInspector(
                ImmutableList.copyOf(transform(testColumns, TestColumn::getName)),
                ImmutableList.copyOf(transform(testColumns, TestColumn::getObjectInspector)));

        Object row = objectInspector.create();

        List<StructField> fields = ImmutableList.copyOf(objectInspector.getAllStructFieldRefs());

        for (int rowNumber = 0; rowNumber < numRows; rowNumber++) {
            for (int i = 0; i < testColumns.size(); i++) {
                Object writeValue = testColumns.get(i).getWriteValue();
                if (writeValue instanceof Slice) {
                    writeValue = ((Slice) writeValue).getBytes();
                }
                objectInspector.setStructFieldData(row, fields.get(i), writeValue);
            }

            Writable record = serDe.serialize(row, objectInspector);
            recordWriter.write(record);
        }
    }
    finally {
        recordWriter.close(false);
    }

    Path path = new Path(filePath);
    path.getFileSystem(new Configuration()).setVerifyChecksum(true);
    File file = new File(filePath);
    return new FileSplit(path, 0, file.length(), new String[0]);
}
 
开发者ID:y-lan,项目名称:presto,代码行数:76,代码来源:AbstractTestHiveFileFormats.java


注:本文中的org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter.write方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。