Java DBInputFormat类代码示例

import org.apache.hadoop.mapreduce.lib.db.DBInputFormat; //导入依赖的package包/类
public static void main(String[] args) throws IOException, Exception {  
 Configuration conf = new Configuration();
 Job job = new Job(conf);
 job.setJarByClass(DBToFileMapReduce.class);  
 DistributedCache.addFileToClassPath(new Path(  
       "/lib/mysql-connector-java-5.1.0-bin.jar"), conf);  
    
 job.setMapperClass(DBInputMapper.class);  
 job.setReducerClass(Reducer.class); 
  
 job.setMapOutputKeyClass(LongWritable.class);  
 job.setMapOutputValueClass(Text.class);  
 job.setOutputKeyClass(LongWritable.class);  
 job.setOutputValueClass(Text.class);  
    
 job.setInputFormatClass(DBInputFormat.class); 
 
 Path outputPath = new Path("/hua01");
 FileOutputFormat.setOutputPath(job, outputPath);  
 outputPath.getFileSystem(conf).delete(outputPath, true);
 
 DBConfiguration.configureDB(conf, "com.mysql.jdbc.Driver",  
       "jdbc:mysql://192.168.3.244:3306/hadoop", "hua", "hadoop");  
 String[] fields = { "id", "name" };  
 DBInputFormat.setInput(job, StudentinfoRecord.class, "studentinfo", null, "id", fields);   

 job.waitForCompletion(true); 
}
 

import org.apache.hadoop.mapreduce.lib.db.DBInputFormat; //导入依赖的package包/类
@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {
  
  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;
  
  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }
  
  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = new Job(conf);
      
  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);
  
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}
 

import org.apache.hadoop.mapreduce.lib.db.DBInputFormat; //导入依赖的package包/类
@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {
  
  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;
  
  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }
  
  initialize(driverClassName, url);
  Configuration conf = getConf();

  DBConfiguration.configureDB(conf, driverClassName, url);

  Job job = Job.getInstance(conf);
      
  job.setJobName("Count Pageviews of URLs");
  job.setJarByClass(DBCountPageView.class);
  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBInputFormat.setInput(job, AccessRecord.class, "HAccess"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);
  
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);
  int ret;
  try {
    ret = job.waitForCompletion(true) ? 0 : 1;
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return ret;
}
 

import org.apache.hadoop.mapreduce.lib.db.DBInputFormat; //导入依赖的package包/类
/**
 * Validate whether class is {@link DBInputFormat} or a subclass.
 *
 * @param clazz the class to validate
 * @param ctx the context to throw the error in if class is invalid
 * @param scope the scope to throw the error in if class is invalid
 */
public static void validateClass(final Class<?> clazz, final Context ctx, final Scriptable scope) {
    if (!ReflectionUtils.isClassOrSubclass(DBInputFormat.class, clazz)) {
        throw Utils.makeError(ctx, scope, LembosMessages.makeInvalidClassErrorMessage(DBInputFormat.class,
                                                                                      clazz));
    }
}
 

import org.apache.hadoop.mapreduce.lib.db.DBInputFormat; //导入依赖的package包/类
/**
 * Java wrapper for {@link DBInputFormat#setInput(org.apache.hadoop.mapreduce.Job, Class, String, String)} and
 * {@link DBInputFormat#setInput(org.apache.hadoop.mapreduce.Job, Class, String, String, String, String...)}.
 *
 * @param ctx the JavaScript context
 * @param thisObj the 'this' object
 * @param args the function arguments
 * @param func the function being called
 */
@JSStaticFunction
public static void setInput(final Context ctx, final Scriptable thisObj, final Object[] args,
                                final Function func) {
    DBInputFormatHelper.setInput(DBInputFormat.class, ctx, thisObj, args, func);
}