本文整理汇总了Java中org.apache.hadoop.mapreduce.lib.output.FileOutputFormat.getCompressOutput方法的典型用法代码示例。如果您正苦于以下问题：Java FileOutputFormat.getCompressOutput方法的具体用法？Java FileOutputFormat.getCompressOutput怎么用？Java FileOutputFormat.getCompressOutput使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.hadoop.mapreduce.lib.output.FileOutputFormat的用法示例。
在下文中一共展示了FileOutputFormat.getCompressOutput方法的2个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: configureDataFileWriter
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; //导入方法依赖的package包/类
static <T> void configureDataFileWriter(DataFileWriter<T> writer,
TaskAttemptContext context) throws UnsupportedEncodingException {
if (FileOutputFormat.getCompressOutput(context)) {
int level = context.getConfiguration()
.getInt(DEFLATE_LEVEL_KEY, DEFAULT_DEFLATE_LEVEL);
String codecName = context.getConfiguration()
.get(org.apache.avro.mapred.AvroJob.OUTPUT_CODEC, DEFLATE_CODEC);
CodecFactory factory =
codecName.equals(DEFLATE_CODEC) ? CodecFactory.deflateCodec(level)
: CodecFactory.fromString(codecName);
writer.setCodec(factory);
}
writer.setSyncInterval(context.getConfiguration()
.getInt(SYNC_INTERVAL_KEY, DEFAULT_SYNC_INTERVAL));
// copy metadata from job
for (Map.Entry<String, String> e : context.getConfiguration()) {
if (e.getKey().startsWith(org.apache.avro.mapred.AvroJob.TEXT_PREFIX)) {
writer.setMeta(e.getKey()
.substring(org.apache.avro.mapred.AvroJob.TEXT_PREFIX.length()),
e.getValue());
}
if (e.getKey().startsWith(org.apache.avro.mapred.AvroJob.BINARY_PREFIX)) {
writer.setMeta(e.getKey()
.substring(org.apache.avro.mapred.AvroJob.BINARY_PREFIX.length()),
URLDecoder.decode(e.getValue(), "ISO-8859-1").getBytes("ISO-8859-1"));
}
}
}
示例2: setup
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; //导入方法依赖的package包/类
@Override
protected void setup(Context context)
throws IOException, InterruptedException {
if (!context.nextKey()
|| context.getCurrentKey().getType() != GridmixKey.REDUCE_SPEC) {
throw new IOException("Missing reduce spec");
}
long outBytes = 0L;
long outRecords = 0L;
long inRecords = 0L;
ResourceUsageMetrics metrics = new ResourceUsageMetrics();
for (GridmixRecord ignored : context.getValues()) {
final GridmixKey spec = context.getCurrentKey();
inRecords += spec.getReduceInputRecords();
outBytes += spec.getReduceOutputBytes();
outRecords += spec.getReduceOutputRecords();
if (spec.getReduceResourceUsageMetrics() != null) {
metrics = spec.getReduceResourceUsageMetrics();
}
}
if (0 == outRecords && inRecords > 0) {
LOG.info("Spec output bytes w/o records. Using input record count");
outRecords = inRecords;
}
// enable gridmix reduce output record for compression
Configuration conf = context.getConfiguration();
if (CompressionEmulationUtil.isCompressionEmulationEnabled(conf)
&& FileOutputFormat.getCompressOutput(context)) {
float compressionRatio =
CompressionEmulationUtil
.getJobOutputCompressionEmulationRatio(conf);
LOG.info("GridMix is configured to use a compression ratio of "
+ compressionRatio + " for the reduce output data.");
val.setCompressibility(true, compressionRatio);
// Set the actual output data size to make sure that the actual output
// data size is same after compression
outBytes /= compressionRatio;
}
factory =
new AvgRecordFactory(outBytes, outRecords,
context.getConfiguration(), 5*1024);
ratio = outRecords / (1.0 * inRecords);
acc = 0.0;
matcher = new ResourceUsageMatcherRunner(context, metrics);
// start the status reporter thread
reporter = new StatusReporter(context, matcher);
reporter.start();
}