Java AvroWrapper类代码示例

import org.apache.avro.mapred.AvroWrapper; //导入依赖的package包/类
@Override
public boolean nextKeyValue() throws IOException, InterruptedException {
  if (!reader.hasNext() || reader.pastSync(end)) {
    key = null;
    value = null;
    return false;
  }
  if (key == null) {
    key = new AvroWrapper<T>();
  }
  if (value == null) {
    value = NullWritable.get();
  }
  key.datum(reader.next(key.datum()));
  return true;
}
 

import org.apache.avro.mapred.AvroWrapper; //导入依赖的package包/类
public void reduce(Text key,
                   Iterator<DoubleWritable> values,
                   OutputCollector<AvroWrapper<StockAvg>,
                       NullWritable> output,
                   Reporter reporter) throws IOException {

  Mean mean = new Mean();
  while (values.hasNext()) {
    mean.increment(values.next().get());
  }
  StockAvg avg = new StockAvg();
  avg.setSymbol(key.toString());
  avg.setAvg(mean.getResult());
  output.collect(new AvroWrapper<StockAvg>(avg),
      NullWritable.get());
}
 

import org.apache.avro.mapred.AvroWrapper; //导入依赖的package包/类
@Override
@SuppressWarnings("unchecked")
public void write(AvroWrapper<Pair<Utf8, T>> record, NullWritable n) throws IOException, InterruptedException {
  String key = record.datum().key().toString();
  if (!key.equals(currentKey)) {
    if (currentWriter != null) {
      currentWriter.close(null); // TaskAttemptContext not used for close
    }
    currentKey = key;
    Path dir = new Path(basePath, key);
    FileSystem fs = dir.getFileSystem(conf);
    if (!fs.exists(dir)) {
      fs.mkdirs(dir);
    }
    currentWriter = (RecordWriter<Void, T>)
        new AvroParquetFileTarget.CrunchAvroParquetOutputFormat().getRecordWriter(conf,
            new Path(dir, uniqueFileName),
            AvroParquetFileTarget.CrunchAvroParquetOutputFormat.getCompression(conf));
  }
  currentWriter.write(null, record.datum().value());
}
 

import org.apache.avro.mapred.AvroWrapper; //导入依赖的package包/类
public void map(LongWritable key, Text value, OutputCollector<AvroWrapper, NullWritable> oc, Reporter rprtr) throws IOException {
    String[] tokens = value.toString().split("\t");
    String cookie = tokens[0];
    String page = tokens[1];
    String date = tokens[2];
    String time = tokens[3];
    String formatedDate = date + ":" + time;
    Date timestamp = null;
    try {
        timestamp = dateFormatter.parse(formatedDate);
    } catch(ParseException ex) {
        //ignore records with invalid dates
        return;
    }
    String ip = tokens[4];
    
    weblogRecord.setCookie(cookie);
    weblogRecord.setDate(timestamp);
    weblogRecord.setIp(ip);
    weblogRecord.setPage(page);
    outputRecord.datum(weblogRecord);
    oc.collect(outputRecord, NullWritable.get());
}
 

import org.apache.avro.mapred.AvroWrapper; //导入依赖的package包/类
@Override
protected void setup(Context context) throws IOException, InterruptedException {
  wrapper = new AvroWrapper<GenericRecord>();
  schema = AvroJob.getOutputSchema(context.getConfiguration());
  bigDecimalFormatString = context.getConfiguration().getBoolean(
      ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT, ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT_DEFAULT);
}
 

import org.apache.avro.mapred.AvroWrapper; //导入依赖的package包/类
@Override
public RecordWriter<AvroWrapper<T>, NullWritable> getRecordWriter(
  TaskAttemptContext context) throws IOException, InterruptedException {

  boolean isMapOnly = context.getNumReduceTasks() == 0;
  Schema schema =
    isMapOnly ? AvroJob.getMapOutputSchema(context.getConfiguration())
      : AvroJob.getOutputSchema(context.getConfiguration());

  final DataFileWriter<T> WRITER =
    new DataFileWriter<T>(new ReflectDatumWriter<T>());

  configureDataFileWriter(WRITER, context);

  Path path = getDefaultWorkFile(context, EXT);
  WRITER.create(schema,
    path.getFileSystem(context.getConfiguration()).create(path));

  return new RecordWriter<AvroWrapper<T>, NullWritable>() {
    @Override
    public void write(AvroWrapper<T> wrapper, NullWritable ignore)
      throws IOException {
      WRITER.append(wrapper.datum());
    }

    @Override
    public void close(TaskAttemptContext taskAttemptContext)
      throws IOException, InterruptedException {
      WRITER.close();
    }
  };
}
 

import org.apache.avro.mapred.AvroWrapper; //导入依赖的package包/类
@Override
public RecordReader<AvroWrapper<T>, NullWritable> createRecordReader(
    InputSplit split, TaskAttemptContext context) throws IOException,
    InterruptedException {
  context.setStatus(split.toString());
  return new AvroRecordReader<T>();
}
 

import org.apache.avro.mapred.AvroWrapper; //导入依赖的package包/类
public void map(LongWritable key,
                StockDbWritable value,
                OutputCollector<AvroWrapper<Stock>, NullWritable> output,
                Reporter reporter) throws IOException {
  output.collect(
      new AvroWrapper<Stock>(writableToAvro(value)),
      NullWritable.get());
}
 

import org.apache.avro.mapred.AvroWrapper; //导入依赖的package包/类
public void map(AvroWrapper<Stock> key,
                NullWritable value,
                OutputCollector<Text, DoubleWritable> output,
                Reporter reporter) throws IOException {
  output.collect(new Text(key.datum().getSymbol().toString()),
      new DoubleWritable(key.datum().getOpen()));
}
 

import org.apache.avro.mapred.AvroWrapper; //导入依赖的package包/类
public void map(AvroWrapper<GenericRecord> key,
                NullWritable value,
                OutputCollector<Text, Text> output,
                Reporter reporter) throws IOException {
  outKey.set(
    key.datum().get(SmallFilesWrite.FIELD_FILENAME).toString());
  outValue.set(DigestUtils.md5Hex(
        ((ByteBuffer) key.datum().get(SmallFilesWrite.FIELD_CONTENTS))
          .array()));

  output.collect(outKey, outValue);
}
 

import org.apache.avro.mapred.AvroWrapper; //导入依赖的package包/类
@Override
public void reduce(NullWritable key, Iterator<BloomFilter> values,
                   OutputCollector<AvroWrapper<GenericRecord>,
                       NullWritable> output,
                   Reporter reporter) throws IOException {
  while (values.hasNext()) {
    BloomFilter bf = values.next();
    filter.or(bf);
    System.out.println(filter);
  }
  collector = output;
}
 

import org.apache.avro.mapred.AvroWrapper; //导入依赖的package包/类
@Override
public void close() throws IOException {
  System.out.println(filter);
  if (collector != null) {
    collector.collect(
        new AvroWrapper<GenericRecord>(
            AvroBytesRecord.toGenericRecord(filter)),
        NullWritable.get());
  }
}
 

import org.apache.avro.mapred.AvroWrapper; //导入依赖的package包/类
@Override
public boolean next(AvroWrapper<T> wrapper, NullWritable ignore)
    throws IOException {
  try {
    if (!records_.hasNext()) return false;
    wrapper.datum(records_.next());
    return true;
  } catch (RecordServiceException e) {
    throw new IOException("Could not get next record.", e);
  }
}
 

import org.apache.avro.mapred.AvroWrapper; //导入依赖的package包/类
@Override
public RecordWriter<AvroWrapper<Pair<Utf8, T>>, NullWritable> getRecordWriter(TaskAttemptContext taskAttemptContext)
    throws IOException, InterruptedException {
  Configuration conf = taskAttemptContext.getConfiguration();
  Path basePath = new Path(getOutputPath(taskAttemptContext), conf.get("mapreduce.output.basename", "part"));
  return new AvroParquetFilePerKeyRecordWriter<T>(basePath,
      getUniqueFile(taskAttemptContext, "part", ".parquet"), conf);
}
 

import org.apache.avro.mapred.AvroWrapper; //导入依赖的package包/类
@Override
public void write(K2 k, V2 v) throws IOException {

    GenericRecord record = fromText(k.toString() + "\t" + v.toString(), schema);
    AvroWrapper<GenericRecord> wrapper = new AvroWrapper<GenericRecord>(record);
    writer.append(wrapper.datum());
}