Java DBConfiguration类代码示例

import org.apache.hadoop.mapred.lib.db.DBConfiguration; //导入依赖的package包/类
/**
 * 
 * test DBRecordReader. This reader should creates keys, values, know about position.. 
 */
@SuppressWarnings("unchecked")
@Test (timeout = 5000)
public void testDBRecordReader() throws Exception {

  JobConf job = mock(JobConf.class);
  DBConfiguration dbConfig = mock(DBConfiguration.class);
  String[] fields = { "field1", "filed2" };

  @SuppressWarnings("rawtypes")
  DBRecordReader reader = new DBInputFormat<NullDBWritable>().new DBRecordReader(
      new DBInputSplit(),  NullDBWritable.class, job,
      DriverForTest.getConnection(), dbConfig, "condition", fields, "table");
  LongWritable key = reader.createKey();
  assertEquals(0, key.get());
  DBWritable value = reader.createValue();
  assertEquals(
      "org.apache.hadoop.mapred.lib.db.DBInputFormat$NullDBWritable", value
          .getClass().getName());
  assertEquals(0, reader.getPos());
  assertFalse(reader.next(key, value));

}
 

import org.apache.hadoop.mapred.lib.db.DBConfiguration; //导入依赖的package包/类
private void configureJobProperties(TableDesc tableDesc, Map<String, String> jobProperties) {
    if(LOG.isDebugEnabled()) {
        LOG.debug("tabelDesc: " + tableDesc);
        LOG.debug("jobProperties: " + jobProperties);
    }

    String tblName = tableDesc.getTableName();
    Properties tblProps = tableDesc.getProperties();
    String columnNames = tblProps.getProperty(Constants.LIST_COLUMNS);
    jobProperties.put(DBConfiguration.INPUT_CLASS_PROPERTY, DbRecordWritable.class.getName());
    jobProperties.put(DBConfiguration.INPUT_TABLE_NAME_PROPERTY, tblName);
    jobProperties.put(DBConfiguration.OUTPUT_TABLE_NAME_PROPERTY, tblName);
    jobProperties.put(DBConfiguration.INPUT_FIELD_NAMES_PROPERTY, columnNames);
    jobProperties.put(DBConfiguration.OUTPUT_FIELD_NAMES_PROPERTY, columnNames);

    for(String key : tblProps.stringPropertyNames()) {
        if(key.startsWith("mapred.jdbc.")) {
            String value = tblProps.getProperty(key);
            jobProperties.put(key, value);
        }
    }
}
 

import org.apache.hadoop.mapred.lib.db.DBConfiguration; //导入依赖的package包/类
private void setupDriver(JobConf configuration) throws Exception {
  configuration.set(DBConfiguration.URL_PROPERTY, "testUrl");
  DriverManager.registerDriver(new DriverForTest());
  configuration.set(DBConfiguration.DRIVER_CLASS_PROPERTY,
      DriverForTest.class.getCanonicalName());
}
 

import org.apache.hadoop.mapred.lib.db.DBConfiguration; //导入依赖的package包/类
/** 
 * test configuration for db. should works DBConfiguration.* parameters. 
 */
@Test (timeout = 5000)
public void testSetInput() {
  JobConf configuration = new JobConf();

  String[] fieldNames = { "field1", "field2" };
  DBInputFormat.setInput(configuration, NullDBWritable.class, "table",
      "conditions", "orderBy", fieldNames);
  assertEquals(
      "org.apache.hadoop.mapred.lib.db.DBInputFormat$NullDBWritable",
      configuration.getClass(DBConfiguration.INPUT_CLASS_PROPERTY, null)
          .getName());
  assertEquals("table",
      configuration.get(DBConfiguration.INPUT_TABLE_NAME_PROPERTY, null));

  String[] fields = configuration
      .getStrings(DBConfiguration.INPUT_FIELD_NAMES_PROPERTY);
  assertEquals("field1", fields[0]);
  assertEquals("field2", fields[1]);

  assertEquals("conditions",
      configuration.get(DBConfiguration.INPUT_CONDITIONS_PROPERTY, null));
  assertEquals("orderBy",
      configuration.get(DBConfiguration.INPUT_ORDER_BY_PROPERTY, null));

  configuration = new JobConf();

  DBInputFormat.setInput(configuration, NullDBWritable.class, "query",
      "countQuery");
  assertEquals("query", configuration.get(DBConfiguration.INPUT_QUERY, null));
  assertEquals("countQuery",
      configuration.get(DBConfiguration.INPUT_COUNT_QUERY, null));
  
  JobConf jConfiguration = new JobConf();
  DBConfiguration.configureDB(jConfiguration, "driverClass", "dbUrl", "user",
      "password");
  assertEquals("driverClass",
      jConfiguration.get(DBConfiguration.DRIVER_CLASS_PROPERTY));
  assertEquals("dbUrl", jConfiguration.get(DBConfiguration.URL_PROPERTY));
  assertEquals("user", jConfiguration.get(DBConfiguration.USERNAME_PROPERTY));
  assertEquals("password",
      jConfiguration.get(DBConfiguration.PASSWORD_PROPERTY));
  jConfiguration = new JobConf();
  DBConfiguration.configureDB(jConfiguration, "driverClass", "dbUrl");
  assertEquals("driverClass",
      jConfiguration.get(DBConfiguration.DRIVER_CLASS_PROPERTY));
  assertEquals("dbUrl", jConfiguration.get(DBConfiguration.URL_PROPERTY));
  assertNull(jConfiguration.get(DBConfiguration.USERNAME_PROPERTY));
  assertNull(jConfiguration.get(DBConfiguration.PASSWORD_PROPERTY));
}
 

import org.apache.hadoop.mapred.lib.db.DBConfiguration; //导入依赖的package包/类
@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {
  
  String driverClassName = DRIVER_CLASS;
  String url = DB_URL;
  
  if(args.length > 1) {
    driverClassName = args[0];
    url = args[1];
  }
  
  initialize(driverClassName, url);

  JobConf job = new JobConf(getConf(), DBCountPageView.class);
      
  job.setJobName("Count Pageviews of URLs");

  job.setMapperClass(PageviewMapper.class);
  job.setCombinerClass(LongSumReducer.class);
  job.setReducerClass(PageviewReducer.class);

  DBConfiguration.configureDB(job, driverClassName, url);
  
  DBInputFormat.setInput(job, AccessRecord.class, "Access"
      , null, "url", AccessFieldNames);

  DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);
  
  job.setMapOutputKeyClass(Text.class);
  job.setMapOutputValueClass(LongWritable.class);

  job.setOutputKeyClass(PageviewRecord.class);
  job.setOutputValueClass(NullWritable.class);

  try {
    JobClient.runJob(job);
    
    boolean correct = verify();
    if(!correct) {
      throw new RuntimeException("Evaluation was not correct!");
    }
  } finally {
    shutdown();    
  }
  return 0;
}
 

import org.apache.hadoop.mapred.lib.db.DBConfiguration; //导入依赖的package包/类
/**
 * The MapReduce driver - setup and launch the job.
 *
 * @param args the command-line arguments
 * @return the process exit code
 * @throws Exception if something goes wrong
 */
public int run(final String[] args) throws Exception {

  Cli cli = Cli.builder().setArgs(args).addOptions(CliCommonOpts.OutputFileOption.values()).build();
  int result = cli.runCmd();

  if (result != 0) {
    return result;
  }

  Path output = new Path(cli.getArgValueAsString(CliCommonOpts.OutputFileOption.OUTPUT));

  Configuration conf = super.getConf();

  DBConfiguration.configureDB(conf, "com.mysql.jdbc.Driver",
      "jdbc:mysql://localhost/sqoop_test" +
          "?user=hip_sqoop_user&password=password");

  JobConf job = new JobConf(conf);
  job.setJarByClass(DBImportMapReduce.class);

  job.setInputFormat(DBInputFormat.class);
  job.setOutputFormat(AvroOutputFormat.class);
  AvroJob.setOutputSchema(job, Stock.SCHEMA$);
  job.set(AvroJob.OUTPUT_CODEC, SnappyCodec.class.getName());

  job.setMapperClass(Map.class);

  job.setNumMapTasks(4);
  job.setNumReduceTasks(0);

  job.setMapOutputKeyClass(AvroWrapper.class);
  job.setMapOutputValueClass(NullWritable.class);

  job.setOutputKeyClass(AvroWrapper.class);
  job.setOutputValueClass(NullWritable.class);

  FileOutputFormat.setOutputPath(job, output);

  DBInputFormat.setInput(
      job,
      StockDbWritable.class,
      "select * from stocks",
      "SELECT COUNT(id) FROM stocks");

  RunningJob runningJob = JobClient.runJob(job);

  return runningJob.isSuccessful() ? 0 : 1;
}
 

import org.apache.hadoop.mapred.lib.db.DBConfiguration; //导入依赖的package包/类
@Override
//Usage DBCountPageView [driverClass dburl]
public int run(String[] args) throws Exception {

    String driverClassName = DRIVER_CLASS;
    String url = DB_URL;

    if (args.length > 1) {
        driverClassName = args[0];
        url = args[1];
    }

    initialize(driverClassName, url);

    JobConf job = new JobConf(getConf(), DBCountPageView.class);

    job.setJobName("Count Pageviews of URLs");

    job.setMapperClass(PageviewMapper.class);
    job.setCombinerClass(LongSumReducer.class);
    job.setReducerClass(PageviewReducer.class);

    DBConfiguration.configureDB(job, driverClassName, url);

    DBInputFormat.setInput(job, AccessRecord.class, "Access", null, "url", AccessFieldNames);

    DBOutputFormat.setOutput(job, "Pageview", PageviewFieldNames);

    job.setMapOutputKeyClass(Text.class);
    job.setMapOutputValueClass(LongWritable.class);

    job.setOutputKeyClass(PageviewRecord.class);
    job.setOutputValueClass(NullWritable.class);

    try {
        JobClient.runJob(job);

        boolean correct = verify();
        if (!correct) {
            throw new RuntimeException("Evaluation was not correct!");
        }
    } finally {
        shutdown();
    }
    return 0;
}