Java AvroGenericRecordWritable类代码示例

import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable; //导入依赖的package包/类
@Override
protected void map(RecordKey key, AvroGenericRecordWritable value,
                   org.apache.hadoop.mapreduce.Mapper<RecordKey, AvroGenericRecordWritable, RecordKey, AvroValue<Record>>.Context
                     context)
  throws IOException, InterruptedException {
  if (key.isValid()) {
    key.setHash(recordValue.hashCode());
    recordWrapped.datum((Record) value.getRecord());
    context.write(recordKey, recordWrapped);
  } else {
    context.getCounter(RecordCounter.RECORDS).increment(1);
    context.getCounter(RecordCounter.RECORDS_MALFORMED).increment(1);
    textValue.set(key.getSource());
    string.setLength(0);
    multipleOutputs.write(OUTPUT_TEXT, NullWritable.get(), textValue,
      string.append(MALFORMED_PATH_PREFIX).append(key.getBatch()).toString());
  }
}
 

import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable; //导入依赖的package包/类
public AvroGenericRecordWritable decode(byte[] payload) {
	try {
		MessageDecoderHelper helper = new MessageDecoderHelper(registry,
				topicName, payload).invoke();
     DatumReader<Record> reader = new GenericDatumReader<Record>(helper.getTargetSchema());

     log.debug("Trying to read kafka payload");
     log.debug("buffer: " + helper.getBuffer());
     log.debug("start: " + helper.getStart());
     log.debug("length: " + helper.getLength());
     log.debug("target schema: " + helper.getTargetSchema());
     log.debug("schema: " + helper.getSchema());
	  GenericRecord record = reader.read(null, decoderFactory.binaryDecoder(helper.getBuffer().array(),
         helper.getStart(), helper.getLength(), null));
     log.debug("Read kafka payload as " + record);

     AvroGenericRecordWritable grw = new AvroGenericRecordWritable(record);
     grw.setFileSchema(latestSchema);

     return grw;

	} catch (IOException e) {
		throw new MessageDecoderException(e);
	}
}
 

import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable; //导入依赖的package包/类
/**
 * Get a {@link SequenceFileInputFormat} from a <code>type</code>
 *
 * @param type the type to lookup
 * @return the {@link SequenceFileInputFormat} instance
 */
public static Class<? extends SequenceFileInputFormat<RecordKey, AvroGenericRecordWritable>> getRecordSequenceInputFormat(String type)
  throws IOException {
  if (!RECORD_INPUT_FORMATS.containsKey(type)) {
    throw new IOException("Could not find [RecordSequenceInputFormat] for type [" + type + "]");
  }
  return RECORD_INPUT_FORMATS.get(type);
}
 

import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable; //导入依赖的package包/类
@Override
@SuppressWarnings({"unchecked", "rawtypes"})
protected void setup(
  org.apache.hadoop.mapreduce.Mapper<RecordKey, AvroGenericRecordWritable, RecordKey, AvroValue<Record>>.Context context)
  throws IOException, InterruptedException {
  multipleOutputs = new MultipleOutputs(context);
}
 

import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable; //导入依赖的package包/类
private AvroGenericRecordWritable getWrappedRecord(String topicName, byte[] payload) throws IOException {
  AvroGenericRecordWritable r = null;
  try {
    r = decoder.decode(payload);
  } catch (Exception e) {
    if (!skipSchemaErrors) {
      throw new IOException(e);
    }
  }
  return r;
}
 

import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable; //导入依赖的package包/类
@Override
public Iterable<AvroGenericRecordWritable> convertRecord(Schema outputSchema, GenericRecord inputRecord,
    WorkUnitState workUnit) throws DataConversionException {
  AvroGenericRecordWritable avroWritable = new AvroGenericRecordWritable();
  avroWritable.setRecord(inputRecord);
  avroWritable.setFileSchema(outputSchema);
  avroWritable.setRecordReaderID(this.uid);
  return Lists.newArrayList(avroWritable);
}
 

import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable; //导入依赖的package包/类
@Override
public AvroGenericRecordWritable getCurrentValue() throws IOException, InterruptedException {
  return recordWriteable;
}
 

import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable; //导入依赖的package包/类
@Override
public RecordReader<RecordKey, AvroGenericRecordWritable> createRecordReader(InputSplit split, TaskAttemptContext context)
  throws IOException {
  return new RecordReaderTextCsv(split, context);
}
 

import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable; //导入依赖的package包/类
@Override
public RecordReader<RecordKey, AvroGenericRecordWritable> createRecordReader(InputSplit split, TaskAttemptContext context)
  throws IOException {
  return new RecordReaderSequenceCsv();
}
 

import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable; //导入依赖的package包/类
@Override
public RecordReader<RecordKey, AvroGenericRecordWritable> createRecordReader(InputSplit split, TaskAttemptContext context)
  throws IOException {
  return new RecordReaderSequenceXml();
}
 

import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable; //导入依赖的package包/类
@Override
public RecordReader<RecordKey, AvroGenericRecordWritable> createRecordReader(InputSplit split, TaskAttemptContext context)
  throws IOException {
  return new RecordReaderTextXml(split, context);
}
 

import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable; //导入依赖的package包/类
@Override
protected void cleanup(
  org.apache.hadoop.mapreduce.Mapper<RecordKey, AvroGenericRecordWritable, RecordKey, AvroValue<Record>>.Context context)
  throws IOException, InterruptedException {
  multipleOutputs.close();
}
 

import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable; //导入依赖的package包/类
@Override
public RecordReader<KafkaKey, AvroGenericRecordWritable> getRecordReader(
		InputSplit split, JobConf conf, Reporter reporter) throws IOException {
	return new KafkaRecordReader(split, conf, reporter);
}
 

import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable; //导入依赖的package包/类
@Override
public AvroGenericRecordWritable createValue() {
  return new AvroGenericRecordWritable();
}
 

import org.apache.hadoop.hive.serde2.avro.AvroGenericRecordWritable; //导入依赖的package包/类
public abstract AvroGenericRecordWritable decode(M message) ;