本文整理汇总了Java中org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter.write方法的典型用法代码示例。如果您正苦于以下问题:Java RecordWriter.write方法的具体用法?Java RecordWriter.write怎么用?Java RecordWriter.write使用的例子?那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter
的用法示例。
在下文中一共展示了RecordWriter.write方法的4个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: createEmptyFile
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; //导入方法依赖的package包/类
@SuppressWarnings({"rawtypes", "unchecked"})
private static Path createEmptyFile(Path hiveScratchDir,
HiveOutputFormat outFileFormat, JobConf job,
int sequenceNumber, Properties props, boolean dummyRow)
throws IOException, InstantiationException, IllegalAccessException {
// create a dummy empty file in a new directory
String newDir = hiveScratchDir + Path.SEPARATOR + sequenceNumber;
Path newPath = new Path(newDir);
FileSystem fs = newPath.getFileSystem(job);
fs.mkdirs(newPath);
//Qualify the path against the file system. The user configured path might contain default port which is skipped
//in the file status. This makes sure that all paths which goes into PathToPartitionInfo are always listed status
//file path.
newPath = fs.makeQualified(newPath);
String newFile = newDir + Path.SEPARATOR + "emptyFile";
Path newFilePath = new Path(newFile);
RecordWriter recWriter = outFileFormat.getHiveRecordWriter(job, newFilePath,
Text.class, false, props, null);
if (dummyRow) {
// empty files are omitted at CombineHiveInputFormat.
// for meta-data only query, it effectively makes partition columns disappear..
// this could be fixed by other methods, but this seemed to be the most easy (HIVEV-2955)
recWriter.write(new Text("empty")); // written via HiveIgnoreKeyTextOutputFormat
}
recWriter.close(false);
return newPath;
}
示例2: createTestFile
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; //导入方法依赖的package包/类
public static FileSplit createTestFile(String filePath,
HiveOutputFormat<?, ?> outputFormat,
@SuppressWarnings("deprecation") SerDe serDe,
String compressionCodec,
List<TestColumn> testColumns,
int numRows)
throws Exception
{
// filter out partition keys, which are not written to the file
testColumns = ImmutableList.copyOf(filter(testColumns, not(TestColumn::isPartitionKey)));
JobConf jobConf = new JobConf();
ReaderWriterProfiler.setProfilerOptions(jobConf);
Properties tableProperties = new Properties();
tableProperties.setProperty("columns", Joiner.on(',').join(transform(testColumns, TestColumn::getName)));
tableProperties.setProperty("columns.types", Joiner.on(',').join(transform(testColumns, TestColumn::getType)));
serDe.initialize(CONFIGURATION, tableProperties);
if (compressionCodec != null) {
CompressionCodec codec = new CompressionCodecFactory(CONFIGURATION).getCodecByName(compressionCodec);
jobConf.set(COMPRESS_CODEC, codec.getClass().getName());
jobConf.set(COMPRESS_TYPE, SequenceFile.CompressionType.BLOCK.toString());
}
RecordWriter recordWriter = createRecordWriter(new Path(filePath), CONFIGURATION);
try {
SettableStructObjectInspector objectInspector = getStandardStructObjectInspector(
ImmutableList.copyOf(transform(testColumns, TestColumn::getName)),
ImmutableList.copyOf(transform(testColumns, TestColumn::getObjectInspector)));
Object row = objectInspector.create();
List<StructField> fields = ImmutableList.copyOf(objectInspector.getAllStructFieldRefs());
for (int rowNumber = 0; rowNumber < numRows; rowNumber++) {
for (int i = 0; i < testColumns.size(); i++) {
Object writeValue = testColumns.get(i).getWriteValue();
if (writeValue instanceof Slice) {
writeValue = ((Slice) writeValue).getBytes();
}
objectInspector.setStructFieldData(row, fields.get(i), writeValue);
}
Writable record = serDe.serialize(row, objectInspector);
recordWriter.write(record);
if (rowNumber % STRIPE_ROWS == STRIPE_ROWS - 1) {
flushStripe(recordWriter);
}
}
}
finally {
recordWriter.close(false);
}
Path path = new Path(filePath);
path.getFileSystem(CONFIGURATION).setVerifyChecksum(true);
File file = new File(filePath);
return new FileSplit(path, 0, file.length(), new String[0]);
}
示例3: writeLineItems
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; //导入方法依赖的package包/类
public static DataSize writeLineItems(
File outputFile,
HiveOutputFormat<?, ?> outputFormat,
@SuppressWarnings("deprecation") Serializer serializer,
CompressionType compressionType,
List<? extends TpchColumn<?>> columns)
throws Exception
{
RecordWriter recordWriter = createRecordWriter(columns, outputFile, outputFormat, compressionType);
SettableStructObjectInspector objectInspector = getStandardStructObjectInspector(transform(columns, input -> input.getColumnName()), transform(columns, input -> getObjectInspector(input)));
Object row = objectInspector.create();
List<StructField> fields = ImmutableList.copyOf(objectInspector.getAllStructFieldRefs());
for (LineItem lineItem : new LineItemGenerator(1, 1, 1)) {
objectInspector.setStructFieldData(row, fields.get(0), lineItem.getOrderKey());
objectInspector.setStructFieldData(row, fields.get(1), lineItem.getPartKey());
objectInspector.setStructFieldData(row, fields.get(2), lineItem.getSupplierKey());
objectInspector.setStructFieldData(row, fields.get(3), lineItem.getLineNumber());
objectInspector.setStructFieldData(row, fields.get(4), lineItem.getQuantity());
objectInspector.setStructFieldData(row, fields.get(5), lineItem.getExtendedPrice());
objectInspector.setStructFieldData(row, fields.get(6), lineItem.getDiscount());
objectInspector.setStructFieldData(row, fields.get(7), lineItem.getTax());
objectInspector.setStructFieldData(row, fields.get(8), lineItem.getReturnFlag());
objectInspector.setStructFieldData(row, fields.get(9), lineItem.getStatus());
objectInspector.setStructFieldData(row, fields.get(10), lineItem.getShipDate());
objectInspector.setStructFieldData(row, fields.get(11), lineItem.getCommitDate());
objectInspector.setStructFieldData(row, fields.get(12), lineItem.getReceiptDate());
objectInspector.setStructFieldData(row, fields.get(13), lineItem.getShipInstructions());
objectInspector.setStructFieldData(row, fields.get(14), lineItem.getShipMode());
objectInspector.setStructFieldData(row, fields.get(15), lineItem.getComment());
Writable record = serializer.serialize(row, objectInspector);
recordWriter.write(record);
}
recordWriter.close(false);
return getFileSize(outputFile);
}
示例4: createTestFile
import org.apache.hadoop.hive.ql.exec.FileSinkOperator.RecordWriter; //导入方法依赖的package包/类
public static FileSplit createTestFile(String filePath,
HiveOutputFormat<?, ?> outputFormat,
@SuppressWarnings("deprecation") SerDe serDe,
String compressionCodec,
List<TestColumn> testColumns,
int numRows)
throws Exception
{
// filter out partition keys, which are not written to the file
testColumns = ImmutableList.copyOf(filter(testColumns, not(TestColumn::isPartitionKey)));
JobConf jobConf = new JobConf();
ReaderWriterProfiler.setProfilerOptions(jobConf);
Properties tableProperties = new Properties();
tableProperties.setProperty("columns", Joiner.on(',').join(transform(testColumns, TestColumn::getName)));
tableProperties.setProperty("columns.types", Joiner.on(',').join(transform(testColumns, TestColumn::getType)));
serDe.initialize(new Configuration(), tableProperties);
if (compressionCodec != null) {
CompressionCodec codec = new CompressionCodecFactory(new Configuration()).getCodecByName(compressionCodec);
jobConf.set(COMPRESS_CODEC, codec.getClass().getName());
jobConf.set(COMPRESS_TYPE, SequenceFile.CompressionType.BLOCK.toString());
jobConf.set("parquet.compression", compressionCodec);
jobConf.set("parquet.enable.dictionary", "true");
}
RecordWriter recordWriter = outputFormat.getHiveRecordWriter(
jobConf,
new Path(filePath),
Text.class,
compressionCodec != null,
tableProperties,
new Progressable()
{
@Override
public void progress()
{
}
}
);
try {
serDe.initialize(new Configuration(), tableProperties);
SettableStructObjectInspector objectInspector = getStandardStructObjectInspector(
ImmutableList.copyOf(transform(testColumns, TestColumn::getName)),
ImmutableList.copyOf(transform(testColumns, TestColumn::getObjectInspector)));
Object row = objectInspector.create();
List<StructField> fields = ImmutableList.copyOf(objectInspector.getAllStructFieldRefs());
for (int rowNumber = 0; rowNumber < numRows; rowNumber++) {
for (int i = 0; i < testColumns.size(); i++) {
Object writeValue = testColumns.get(i).getWriteValue();
if (writeValue instanceof Slice) {
writeValue = ((Slice) writeValue).getBytes();
}
objectInspector.setStructFieldData(row, fields.get(i), writeValue);
}
Writable record = serDe.serialize(row, objectInspector);
recordWriter.write(record);
}
}
finally {
recordWriter.close(false);
}
Path path = new Path(filePath);
path.getFileSystem(new Configuration()).setVerifyChecksum(true);
File file = new File(filePath);
return new FileSplit(path, 0, file.length(), new String[0]);
}