Java AvroJob.setOutputKeySchema方法代码示例

import org.apache.avro.mapreduce.AvroJob; //导入方法依赖的package包/类
/** Hacked method */
private void setSchema(Job job, Schema keySchema, Schema valSchema) {

  boolean isMaponly = job.getNumReduceTasks() == 0;
  if (keySchema != null) {
    if (isMaponly){
      AvroJob.setMapOutputKeySchema(job, keySchema);
    }
    AvroJob.setOutputKeySchema(job, keySchema);
  }
  if (valSchema != null) {
    if (isMaponly){
      AvroJob.setMapOutputValueSchema(job, valSchema);
    }
    AvroJob.setOutputValueSchema(job, valSchema);
  }

}
 

import org.apache.avro.mapreduce.AvroJob; //导入方法依赖的package包/类
@Test
public void testMapReduce() throws IOException {
  MyAvroReducer reducer = new MyAvroReducer();

  // Configure a job.
  Job job = new Job();
  // We've got to do a little hacking here since mrunit doesn't run exactly like
  // the real hadoop mapreduce framework.
  AvroJob.setMapOutputKeySchema(job, Node.SCHEMA$);
  AvroJob.setOutputKeySchema(job, reducer.getAvroKeyWriterSchema());
  AvroSerialization.setValueWriterSchema(job.getConfiguration(), Node.SCHEMA$);

  // Run the reducer.
  ReduceDriver<Text, AvroValue<Node>, AvroKey<Node>, NullWritable> driver
      = new ReduceDriver<Text, AvroValue<Node>, AvroKey<Node>, NullWritable>();
  driver.setReducer(reducer);
  driver.withConfiguration(job.getConfiguration());
  driver.withInput(new Text("foo"),
      Collections.singletonList(new AvroValue<Node>(new NodeBuilder("bar", 1.0).build())));
  List<Pair<AvroKey<Node>, NullWritable>> output = driver.run();
  assertEquals(1, output.size());
  assertEquals("bar", output.get(0).getFirst().datum().getLabel().toString());
}
 

import org.apache.avro.mapreduce.AvroJob; //导入方法依赖的package包/类
@Test
public void testMapReduce() throws IOException {
  MyNodeReducer reducer = new MyNodeReducer();

  // Configure a job.
  Job job = new Job();
  // We've got to do a little hacking here since mrunit doesn't run exactly like
  // the real hadoop mapreduce framework.
  AvroJob.setMapOutputKeySchema(job, Node.SCHEMA$);
  AvroJob.setOutputKeySchema(job, reducer.getAvroKeyWriterSchema());
  AvroSerialization.setValueWriterSchema(job.getConfiguration(), Node.SCHEMA$);

  ReduceDriver<Text, AvroValue<Node>, AvroKey<Node>, NullWritable> driver
      = new ReduceDriver<Text, AvroValue<Node>, AvroKey<Node>, NullWritable>();
  driver.setReducer(reducer);
  driver.withConfiguration(job.getConfiguration());
  driver.withInput(
      new Text("foo"),
      Collections.singletonList(new AvroValue<Node>(new NodeBuilder("bar", 1.0).build())));
  List<Pair<AvroKey<Node>, NullWritable>> output = driver.run();
  assertEquals(1, output.size());
  assertEquals("bar", output.get(0).getFirst().datum().getLabel().toString());
}
 

import org.apache.avro.mapreduce.AvroJob; //导入方法依赖的package包/类
public boolean runMapReduce(final Job job, Path inputPath, Path outputPath) throws Exception {
    FileInputFormat.setInputPaths(job, inputPath);
    job.setInputFormatClass(AvroKeyInputFormat.class);
    AvroJob.setInputKeySchema(job, Weather.SCHEMA$);

    job.setMapperClass(SortMapper.class);
    AvroJob.setMapOutputValueSchema(job, Weather.SCHEMA$);
    job.setMapOutputKeyClass(WeatherSubset.class);

    job.setReducerClass(SortReducer.class);
    AvroJob.setOutputKeySchema(job, Weather.SCHEMA$);

    job.setOutputFormatClass(AvroKeyOutputFormat.class);
    FileOutputFormat.setOutputPath(job, outputPath);

    job.setPartitionerClass(WeatherPartitioner.class);
    job.setGroupingComparatorClass(WeatherSubsetGroupingComparator.class);
    job.setSortComparatorClass(WeatherSubsetSortComparator.class);

    return job.waitForCompletion(true);
}
 

import org.apache.avro.mapreduce.AvroJob; //导入方法依赖的package包/类
public boolean runMapReduce(final Job job, Path inputPath, Path outputPath) throws Exception {
    FileInputFormat.setInputPaths(job, inputPath);
    job.setInputFormatClass(AvroKeyInputFormat.class);
    AvroJob.setInputKeySchema(job, WeatherNoIgnore.SCHEMA$);

    job.setMapperClass(SortMapper.class);
    AvroJob.setMapOutputKeySchema(job, WeatherNoIgnore.SCHEMA$);
    AvroJob.setMapOutputValueSchema(job, WeatherNoIgnore.SCHEMA$);

    job.setReducerClass(SortReducer.class);
    AvroJob.setOutputKeySchema(job, WeatherNoIgnore.SCHEMA$);

    job.setOutputFormatClass(AvroKeyOutputFormat.class);
    FileOutputFormat.setOutputPath(job, outputPath);

    return job.waitForCompletion(true);
}
 

import org.apache.avro.mapreduce.AvroJob; //导入方法依赖的package包/类
public boolean runMapReduce(final Job job, Path inputPath, Path outputPath) throws Exception {
    FileInputFormat.setInputPaths(job, inputPath);
    job.setInputFormatClass(AvroKeyInputFormat.class);
    AvroJob.setInputKeySchema(job, Weather.SCHEMA$);

    job.setMapperClass(SortMapper.class);
    AvroJob.setMapOutputKeySchema(job, Weather.SCHEMA$);
    AvroJob.setMapOutputValueSchema(job, Weather.SCHEMA$);

    job.setReducerClass(SortReducer.class);
    AvroJob.setOutputKeySchema(job, Weather.SCHEMA$);

    job.setOutputFormatClass(AvroKeyOutputFormat.class);
    FileOutputFormat.setOutputPath(job, outputPath);

    return job.waitForCompletion(true);
}
 

import org.apache.avro.mapreduce.AvroJob; //导入方法依赖的package包/类
private void configureSchema(Job job) throws IOException {
  Schema newestSchema = getNewestSchemaFromSource(job);
  AvroJob.setInputKeySchema(job, newestSchema);
  AvroJob.setMapOutputKeySchema(job, this.shouldDeduplicate ? getKeySchema(job, newestSchema) : newestSchema);
  AvroJob.setMapOutputValueSchema(job, newestSchema);
  AvroJob.setOutputKeySchema(job, newestSchema);
}
 

import org.apache.avro.mapreduce.AvroJob; //导入方法依赖的package包/类
public boolean createAndSubmitJob() throws IOException, ClassNotFoundException, InterruptedException {
    Configuration configuration = new Configuration(yarnUnit.getConfig());
    configuration.setBoolean("mapred.mapper.new-api", true);
    configuration.setBoolean("mapred.reducer.new-api", true);
    Job job = Job.getInstance(configuration);
    job.setJobName(this.getClass().getSimpleName() + "-job");

    job.setNumReduceTasks(1);

    job.setMapperClass(AvroMapReduce.AvroMapper.class);

    Schema inputSchema = new Schema.Parser().parse(
            MapreduceAvroTest.class.getClassLoader().getResourceAsStream("mapreduce-avro/input.avsc"));
    FileInputFormat.addInputPath(job, new Path(inputPath));
    job.setInputFormatClass(AvroKeyInputFormat.class);
    AvroJob.setInputKeySchema(job, inputSchema);
    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(Text.class);

    job.setReducerClass(AvroMapReduce.AvroReducer.class);

    FileOutputFormat.setOutputPath(job, new Path(outputPath));
    job.setOutputFormatClass(AvroKeyOutputFormat.class);
    AvroJob.setOutputKeySchema(job, new Schema.Parser().parse(
            MapreduceAvroTest.class.getClassLoader().getResourceAsStream("mapreduce-avro/output.avsc")));
    job.setOutputKeyClass(AvroKey.class);
    job.setOutputValueClass(NullWritable.class);

    job.setSpeculativeExecution(false);
    job.setMaxMapAttempts(1); // speed up failures
    return job.waitForCompletion(true);
}
 

import org.apache.avro.mapreduce.AvroJob; //导入方法依赖的package包/类
@Override
public void process(Annotation annotation, Job job, Object target)
		throws ToolException {

	AvroJobInfo avroInfo = (AvroJobInfo)annotation;
	if (avroInfo.inputKeySchema() != AvroDefault.class) {
		AvroJob.setInputKeySchema(job, getSchema(avroInfo.inputKeySchema()));
	}
	if (avroInfo.inputValueSchema() != AvroDefault.class) {
		AvroJob.setInputValueSchema(job, getSchema(avroInfo.inputValueSchema()));
	}

	if (avroInfo.outputKeySchema() != AvroDefault.class) {
		AvroJob.setOutputKeySchema(job, getSchema(avroInfo.outputKeySchema()));
	}
	if (avroInfo.outputValueSchema() != AvroDefault.class) {
		AvroJob.setOutputValueSchema(job, getSchema(avroInfo.outputValueSchema()));
	}

	if (avroInfo.mapOutputKeySchema() != AvroDefault.class) {
		AvroJob.setMapOutputKeySchema(job, getSchema(avroInfo.mapOutputKeySchema()));
	}
	if (avroInfo.mapOutputValueSchema() != AvroDefault.class) {
		AvroJob.setMapOutputValueSchema(job, getSchema(avroInfo.mapOutputValueSchema()));
	}

	AvroSerialization.addToConfiguration(job.getConfiguration());
}
 

import org.apache.avro.mapreduce.AvroJob; //导入方法依赖的package包/类
private Job getJob(Schema avroSchema) {
    
    Job job;
    
    try {
        job = Job.getInstance();
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    
    AvroJob.setOutputKeySchema(job, avroSchema);

    return job;
}
 

import org.apache.avro.mapreduce.AvroJob; //导入方法依赖的package包/类
private static <T> TransformEvaluator<AvroIO.Write.Bound<T>> writeAvro() {
  return new TransformEvaluator<AvroIO.Write.Bound<T>>() {
    @Override
    public void evaluate(AvroIO.Write.Bound<T> transform, EvaluationContext context) {
      Job job;
      try {
        job = Job.getInstance();
      } catch (IOException e) {
        throw new IllegalStateException(e);
      }
      AvroJob.setOutputKeySchema(job, transform.getSchema());
      @SuppressWarnings("unchecked")
      JavaPairRDD<AvroKey<T>, NullWritable> last =
          ((JavaRDDLike<WindowedValue<T>, ?>) context.getInputRDD(transform))
          .map(WindowingHelpers.<T>unwindowFunction())
          .mapToPair(new PairFunction<T, AvroKey<T>, NullWritable>() {
            @Override
            public Tuple2<AvroKey<T>, NullWritable> call(T t) throws Exception {
              return new Tuple2<>(new AvroKey<>(t), NullWritable.get());
            }
          });
      ShardTemplateInformation shardTemplateInfo =
          new ShardTemplateInformation(transform.getNumShards(),
          transform.getShardTemplate(), transform.getFilenamePrefix(),
          transform.getFilenameSuffix());
      writeHadoopFile(last, job.getConfiguration(), shardTemplateInfo,
          AvroKey.class, NullWritable.class, TemplatedAvroKeyOutputFormat.class);
    }
  };
}
 

import org.apache.avro.mapreduce.AvroJob; //导入方法依赖的package包/类
public int run(String[] args) throws Exception {
  org.apache.log4j.BasicConfigurator.configure();

  if (args.length != 2) {
    System.err.println("Usage: MapReduceAgeCount <input path> <output path>");
    return -1;
  }

  Job job = Job.getInstance(getConf());
  job.setJarByClass(MapReduceAgeCount.class);
  job.setJobName("Age Count");

  // RECORDSERVICE:
  // To read from a table instead of a path, comment out
  // FileInputFormat.setInputPaths() and instead use:
  // FileInputFormat.setInputPaths(job, new Path(args[0]));
  RecordServiceConfig.setInputTable(job.getConfiguration(), null, args[0]);

  // RECORDSERVICE:
  // Use the RecordService version of the AvroKeyValueInputFormat
  job.setInputFormatClass(
      com.cloudera.recordservice.avro.mapreduce.AvroKeyValueInputFormat.class);
  FileOutputFormat.setOutputPath(job, new Path(args[1]));

  job.setMapperClass(AgeCountMapper.class);
  // Set schema for input key and value.
  AvroJob.setInputKeySchema(job, UserKey.getClassSchema());
  AvroJob.setInputValueSchema(job, UserValue.getClassSchema());

  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(IntWritable.class);

  job.setOutputFormatClass(AvroKeyValueOutputFormat.class);
  job.setReducerClass(AgeCountReducer.class);
  AvroJob.setOutputKeySchema(job, Schema.create(Schema.Type.STRING));
  AvroJob.setOutputValueSchema(job, Schema.create(Schema.Type.INT));

  return (job.waitForCompletion(true) ? 0 : 1);
}
 

import org.apache.avro.mapreduce.AvroJob; //导入方法依赖的package包/类
@Override
public int run(String[] args) throws Exception {
  org.apache.log4j.BasicConfigurator.configure();

  if (args.length != 2) {
    System.err.println("Usage: MapReduceColorCount <input path> <output path>");
    return -1;
  }

  Job job = Job.getInstance(getConf());
  job.setJarByClass(MapReduceColorCount.class);
  job.setJobName("Color Count");

  // RECORDSERVICE:
  // To read from a table instead of a path, comment out
  // FileInputFormat.setInputPaths() and instead use:
  //FileInputFormat.setInputPaths(job, new Path(args[0]));
  RecordServiceConfig.setInputTable(job.getConfiguration(), "rs", "users");

  // RECORDSERVICE:
  // Use the RecordService version of the AvroKeyInputFormat
  job.setInputFormatClass(
      com.cloudera.recordservice.avro.mapreduce.AvroKeyInputFormat.class);
  //job.setInputFormatClass(AvroKeyInputFormat.class);

  FileOutputFormat.setOutputPath(job, new Path(args[1]));

  job.setMapperClass(ColorCountMapper.class);
  AvroJob.setInputKeySchema(job, User.getClassSchema());
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(IntWritable.class);

  job.setOutputFormatClass(AvroKeyValueOutputFormat.class);
  job.setReducerClass(ColorCountReducer.class);
  AvroJob.setOutputKeySchema(job, Schema.create(Schema.Type.STRING));
  AvroJob.setOutputValueSchema(job, Schema.create(Schema.Type.INT));

  return (job.waitForCompletion(true) ? 0 : 1);
}
 

import org.apache.avro.mapreduce.AvroJob; //导入方法依赖的package包/类
/**
 * Run the MR2 color count with generic records, and return a map of favorite colors to
 * the number of users.
 */
public static java.util.Map<String, Integer> countColors() throws IOException,
    ClassNotFoundException, InterruptedException {
  String output = TestUtil.getTempDirectory();
  Path outputPath = new Path(output);
  JobConf conf = new JobConf(ColorCount.class);
  conf.setInt("mapreduce.job.reduces", 1);

  Job job = Job.getInstance(conf);
  job.setJarByClass(ColorCount.class);
  job.setJobName("MR2 Color Count With Generic Records");

  RecordServiceConfig.setInputTable(job.getConfiguration(), "rs", "users");
  job.setInputFormatClass(
      com.cloudera.recordservice.avro.mapreduce.AvroKeyInputFormat.class);
  FileOutputFormat.setOutputPath(job, outputPath);

  job.setMapperClass(Map.class);
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(IntWritable.class);

  job.setOutputFormatClass(AvroKeyValueOutputFormat.class);
  job.setReducerClass(Reduce.class);
  AvroJob.setOutputKeySchema(job, Schema.create(Schema.Type.STRING));
  AvroJob.setOutputValueSchema(job, Schema.create(Schema.Type.INT));

  job.waitForCompletion(false);

  // Read the result and return it. Since we set the number of reducers to 1,
  // there is always just one file containing the value.
  SeekableInput input = new FsInput(new Path(output + "/part-r-00000.avro"), conf);
  DatumReader<GenericRecord> reader = new GenericDatumReader<GenericRecord>();
  FileReader<GenericRecord> fileReader = DataFileReader.openReader(input, reader);
  java.util.Map<String, Integer> colorMap = new HashMap<String, Integer>();
  for (GenericRecord datum: fileReader) {
    colorMap.put(datum.get(0).toString(), Integer.parseInt(datum.get(1).toString()));
  }
  return colorMap;
}
 

import org.apache.avro.mapreduce.AvroJob; //导入方法依赖的package包/类
@Override
public void afterPropertiesSet() throws Exception {

    if (avroInputKey != null) {
        AvroJob.setInputKeySchema(job, resolveClass(avroInputKey).newInstance().getSchema());
    }

    if (avroInputValue != null) {
        AvroJob.setInputValueSchema(job, resolveClass(avroInputValue).newInstance().getSchema());
    }

    if (avroMapOutputKey != null) {
        AvroJob.setMapOutputKeySchema(job, resolveClass(avroMapOutputKey).newInstance().getSchema());
    }

    if (avroMapOutputValue != null) {
        Class<? extends IndexedRecord> c = resolveClass(avroMapOutputValue);
        IndexedRecord o = c.newInstance();
        AvroJob.setMapOutputValueSchema(job, o.getSchema());
    }

    if (avroOutputKey != null) {
        AvroJob.setOutputKeySchema(job, resolveClass(avroOutputKey).newInstance().getSchema());
    }

    if (avroOutputValue != null) {
        AvroJob.setOutputValueSchema(job, resolveClass(avroOutputValue).newInstance().getSchema());
    }
}