Java MapDriver类代码示例

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入依赖的package包/类
@Test
public void testTableMapperOutput() throws Exception {
    RyaStatement rya = TestUtils.ryaStatement("x", "subOrganizationOf", "y");
    TripleRowResolver trr = new WholeRowTripleResolver();
    Map<TABLE_LAYOUT,TripleRow> map = trr.serialize(rya);
    TripleRow tr = map.get(TABLE_LAYOUT.SPO);
    byte[] b = new byte[0];
    Key key = new Key(tr.getRow(), tr.getColumnFamily(),
        tr.getColumnQualifier(), b, 1);
    Value val = new Value(b);
    ResourceWritable rw1 = new ResourceWritable();
    ResourceWritable rw2 = new ResourceWritable();
    rw1.set(TestUtils.uri("x"));
    rw2.set(TestUtils.uri("y"));
    new MapDriver<Key, Value, ResourceWritable, Fact>()
        .withMapper(new ForwardChain.TableMapper(schema))
        .withInput(key, val)
        .withOutput(rw1, X_SUB_Y)
        .withOutput(rw2, X_SUB_Y)
        .runTest();
}
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入依赖的package包/类
@Test
public void testRdfMapperOutput() throws Exception {
    RyaStatement rya = TestUtils.ryaStatement("x", "subOrganizationOf", "y");
    RyaStatementWritable rsw = new RyaStatementWritable();
    rsw.setRyaStatement(rya);
    LongWritable l = new LongWritable();
    ResourceWritable rw1 = new ResourceWritable();
    ResourceWritable rw2 = new ResourceWritable();
    rw1.set(TestUtils.uri("x"));
    rw2.set(TestUtils.uri("y"));
    new MapDriver<LongWritable, RyaStatementWritable, ResourceWritable,
            Fact>()
        .withMapper(new ForwardChain.RdfMapper(schema))
        .withInput(l, rsw)
        .withOutput(rw1, X_SUB_Y)
        .withOutput(rw2, X_SUB_Y)
        .runTest();
}
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入依赖的package包/类
@Test
public void testTableMapperOutput() throws Exception {
    RyaStatement rya = TestUtils.ryaStatement("x", "subOrganizationOf", "y");
    TripleRowResolver trr = new WholeRowTripleResolver();
    Map<TABLE_LAYOUT,TripleRow> map = trr.serialize(rya);
    TripleRow tr = map.get(TABLE_LAYOUT.SPO);
    byte[] b = new byte[0];
    Key key = new Key(tr.getRow(), tr.getColumnFamily(),
        tr.getColumnQualifier(), b, 1);
    Value val = new Value(b);
    new MapDriver<Key, Value, Fact, Derivation>()
        .withMapper(new DuplicateElimination.DuplicateTableMapper())
        .withInput(key, val)
        .withOutput(X_SUB_Y, X_SUB_Y.getDerivation())
        .runTest();
}
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入依赖的package包/类
@Test
public void testMapper() throws Exception {
  MorphlineMapper mapper = new MorphlineMapper();
  MapDriver<LongWritable, Text, Text, SolrInputDocumentWritable> mapDriver = MapDriver.newMapDriver(mapper);;

  Configuration config = mapDriver.getConfiguration();
  setupHadoopConfig(config);

  mapDriver.withInput(new LongWritable(0L), new Text("hdfs://localhost/" + DOCUMENTS_DIR + "/sample-statuses-20120906-141433.avro"));

  SolrInputDocument sid = new SolrInputDocument();
  sid.addField("id", "uniqueid1");
  sid.addField("user_name", "user1");
  sid.addField("text", "content of record one");
  SolrInputDocumentWritable sidw = new SolrInputDocumentWritable(sid);

  mapDriver
    .withCacheArchive(solrHomeZip.getAbsolutePath())
    .withOutput(new Text("0"), sidw);
  //mapDriver.runTest();
  List<Pair<Text, SolrInputDocumentWritable>> result = mapDriver.run();
  for (Pair<Text, SolrInputDocumentWritable> p: result) {
    System.out.println(p.getFirst());
    System.out.println(p.getSecond());
  }
}
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入依赖的package包/类
@Before
public void setUp() throws Exception
{
  AggregationMapper mapper = new AggregationMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations","org.apache.hadoop.io.serializer.JavaSerialization,"
                  + "org.apache.hadoop.io.serializer.WritableSerialization");
  configuration.set(AggregationJobConstants.AGG_CONFIG_PATH.toString(), ClassLoader.getSystemResource(CONF_FILE).toString());
  Schema schema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(SCHEMA_FILE));
  setUpAvroSerialization(mapDriver.getConfiguration(), schema);
  metricSchema = new MetricSchema(metricNames, metricTypes);
  AggregationReducer reducer = new AggregationReducer();
  reduceDriver = ReduceDriver.newReduceDriver(reducer);
  configuration = reduceDriver.getConfiguration();
  configuration.set(AggregationJobConstants.AGG_CONFIG_PATH.toString(), ClassLoader.getSystemResource(CONF_FILE).toString());

  TemporaryPath tmpPath = new TemporaryPath();
  outputPath = tmpPath.toString();
  configuration.set(AggregationJobConstants.AGG_OUTPUT_PATH.toString(), outputPath);
 }
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入依赖的package包/类
@BeforeClass
public void setUp() throws IOException
{
  BootstrapMapper mapper = new BootstrapMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration config = mapDriver.getConfiguration();
  config.set("io.serializations","org.apache.hadoop.io.serializer.JavaSerialization,"
      + "org.apache.hadoop.io.serializer.WritableSerialization");
  config.set(StarTreeBootstrapPhaseOneConstants.STAR_TREE_BOOTSTRAP_CONFIG_PATH.toString(), ClassLoader.getSystemResource(CONF_FILE).toString());
  Schema schema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(SCHEMA_FILE));
  setUpAvroSerialization(mapDriver.getConfiguration(), schema);

  Path configPath = new Path(ClassLoader.getSystemResource(CONF_FILE).toString());
  FileSystem fs = FileSystem.get(config);
  StarTreeConfig starTreeConfig = StarTreeConfig.decode(fs.open(configPath));
  starTreeBootstrapConfig = StarTreeBootstrapPhaseOneConfig.fromStarTreeConfig(starTreeConfig);
  thirdEyeRoot = System.getProperty("java.io.tmpdir") ;
  config.set(StarTreeGenerationConstants.STAR_TREE_GEN_OUTPUT_PATH.toString(), thirdEyeRoot + File.separator + "startree_generation");

  StarTreeBootstrapReducer reducer = new StarTreeBootstrapReducer();
  reduceDriver = ReduceDriver.newReduceDriver(reducer);
  config = reduceDriver.getConfiguration();
  config.set(StarTreeBootstrapPhaseOneConstants.STAR_TREE_BOOTSTRAP_CONFIG_PATH.toString(), ClassLoader.getSystemResource(CONF_FILE).toString());
}
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入依赖的package包/类
@BeforeClass
public void setUp() throws IOException
{
  BootstrapPhaseTwoMapper mapper = new BootstrapPhaseTwoMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration config = mapDriver.getConfiguration();
  config.set(StarTreeBootstrapPhaseTwoConstants.STAR_TREE_BOOTSTRAP_PHASE2_CONFIG_PATH.toString(), ClassLoader.getSystemResource(CONF_FILE).toString());

  Path configPath = new Path(ClassLoader.getSystemResource(CONF_FILE).toString());
  FileSystem fs = FileSystem.get(config);
  StarTreeConfig starTreeConfig = StarTreeConfig.decode(fs.open(configPath));
  starTreeBootstrapConfig = StarTreeBootstrapPhaseTwoConfig.fromStarTreeConfig(starTreeConfig);
  thirdEyeRoot = System.getProperty("java.io.tmpdir") ;
  config.set(StarTreeGenerationConstants.STAR_TREE_GEN_OUTPUT_PATH.toString(), thirdEyeRoot + File.separator + "startree_generation");

  BootstrapPhaseTwoReducer reducer = new BootstrapPhaseTwoReducer();
  reduceDriver = ReduceDriver.newReduceDriver(reducer);
  config = reduceDriver.getConfiguration();
  config.set(StarTreeBootstrapPhaseTwoConstants.STAR_TREE_BOOTSTRAP_PHASE2_CONFIG_PATH.toString(), ClassLoader.getSystemResource(CONF_FILE).toString());
  config.set(StarTreeBootstrapPhaseTwoConstants.STAR_TREE_BOOTSTRAP_PHASE2_OUTPUT_PATH.toString(), thirdEyeRoot + File.separator + "startree_bootstrap_phase2");
  config.set(StarTreeGenerationConstants.STAR_TREE_GEN_OUTPUT_PATH.toString(), thirdEyeRoot + File.separator + "startree_generation");
}
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入依赖的package包/类
@BeforeSuite
public void setUp() throws Exception
{
  RollupPhaseTwoMapper mapper = new RollupPhaseTwoMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set(RollupPhaseTwoConstants.ROLLUP_PHASE2_CONFIG_PATH.toString(), ClassLoader.getSystemResource(CONF_FILE).toString());
  configuration.set(RollupPhaseTwoConstants.ROLLUP_PHASE2_ANALYSIS_PATH.toString(), "dummy analysis path");
  Path configPath = new Path(ClassLoader.getSystemResource(CONF_FILE).toString());
  FileSystem fileSystem = FileSystem.get(configuration);
  StarTreeConfig starTreeConfig = StarTreeConfig.decode(fileSystem.open(configPath));
  rollUpConfig = RollupPhaseTwoConfig.fromStarTreeConfig(starTreeConfig);
  RollupPhaseTwoReducer reducer = new RollupPhaseTwoReducer();
  reduceDriver = ReduceDriver.newReduceDriver(reducer);
  configuration = reduceDriver.getConfiguration();
  configuration.set(RollupPhaseTwoConstants.ROLLUP_PHASE2_CONFIG_PATH.toString(), ClassLoader.getSystemResource(CONF_FILE).toString());

}
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入依赖的package包/类
@Before
public void setUp() throws Exception
{
  AnalyzeMapper mapper = new AnalyzeMapper();
  AnalyzeReducer reducer = new AnalyzeReducer();
  mapDriver = MapDriver.newMapDriver(mapper);
  reduceDriver = ReduceDriver.newReduceDriver(reducer);
  Configuration configuration = mapDriver.getConfiguration();
  configuration.set("io.serializations","org.apache.hadoop.io.serializer.JavaSerialization,"
                  + "org.apache.hadoop.io.serializer.WritableSerialization");
  configuration.set(AnalysisJobConstants.ANALYSIS_CONFIG_PATH.toString(), ClassLoader.getSystemResource(CONF_FILE).toString());
  Schema schema = new Schema.Parser().parse(ClassLoader.getSystemResourceAsStream(SCHEMA_FILE));
  setUpAvroSerialization(mapDriver.getConfiguration(), schema);

  configuration = reduceDriver.getConfiguration();
  TemporaryPath tmpDir = new TemporaryPath();
  outputPath = tmpDir.toString();
  configuration.set(AnalysisJobConstants.ANALYSIS_OUTPUT_PATH.toString(), tmpDir.toString());
  configuration.set(AnalysisJobConstants.ANALYSIS_FILE_NAME.toString(), RESULT_FILE);
 }
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入依赖的package包/类
@Before
public void setUp() {

  /*
   * Set up the mapper test harness.
   */
  WordMapper mapper = new WordMapper();
  mapDriver = new MapDriver<LongWritable, Text, Text, IntWritable>();
  mapDriver.setMapper(mapper);

  /*
   * Set up the reducer test harness.
   */
  SumReducer reducer = new SumReducer();
  reduceDriver = new ReduceDriver<Text, IntWritable, Text, IntWritable>();
  reduceDriver.setReducer(reducer);

  /*
   * Set up the mapper/reducer test harness.
   */
  mapReduceDriver = new MapReduceDriver<LongWritable, Text, Text, IntWritable, Text, IntWritable>();
  mapReduceDriver.setMapper(mapper);
  mapReduceDriver.setReducer(reducer);
}
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入依赖的package包/类
@Test
public void shouldMapTagsFromRow() throws IOException {
    //given
    Text input = new Text("<row value=\"blabla\" Tags=\"&lt;java&gt;&lt;spring&gt;\" title=\"Java is awesome\" />");
    List<Pair<RelationKey, MappingType>> map = new ArrayList<Pair<RelationKey, MappingType>>();
    map.add(new Pair(new RelationKey("java", "spring"), MappingType.TAG));
    map.add(new Pair(new RelationKey("spring", "java"), MappingType.TAG));
    //when
    new MapDriver<LongWritable, Text, RelationKey, MappingType>()
            .withMapper(new TechnologiesMapper())
            .withInput(new LongWritable(1l), input)

                    //then
            .withAllOutput(map)
            .runTest();
}
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入依赖的package包/类
@Before
public void setUp() {
    AnalyzerBeansConfiguration analyzerBeansConfiguration = buildAnalyzerBeansConfigurationLocalFS(CSV_FILE_PATH);
    analysisJob = buildAnalysisJob(analyzerBeansConfiguration, CSV_FILE_PATH);
    String analyzerBeansConfigurationDatastores = ConfigurationSerializer
            .serializeAnalyzerBeansConfigurationDataStores(analyzerBeansConfiguration);
    String analysisJobXml = ConfigurationSerializer.serializeAnalysisJobToXml(analyzerBeansConfiguration,
            analysisJob);
    FlatFileMapper flatFileMapper = new FlatFileMapper();
    FlatFileReducer flatFileReducer = new FlatFileReducer();
    mapDriver = MapDriver.newMapDriver(flatFileMapper);
    mapDriver.getConfiguration().set(FlatFileTool.ANALYZER_BEANS_CONFIGURATION_DATASTORES_KEY,
            analyzerBeansConfigurationDatastores);
    mapDriver.getConfiguration().set(FlatFileTool.ANALYSIS_JOB_XML_KEY, analysisJobXml);
    reduceDriver = ReduceDriver.newReduceDriver(flatFileReducer);
    reduceDriver.getConfiguration().set(FlatFileTool.ANALYZER_BEANS_CONFIGURATION_DATASTORES_KEY,
            analyzerBeansConfigurationDatastores);
    reduceDriver.getConfiguration().set(FlatFileTool.ANALYSIS_JOB_XML_KEY, analysisJobXml);
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(flatFileMapper, flatFileReducer);
}
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入依赖的package包/类
@Before
public void setUp() {
	AnalyzerBeansConfiguration analyzerBeansConfiguration = buildAnalyzerBeansConfiguration();
	AnalysisJob analysisJob = buildAnalysisJob(analyzerBeansConfiguration);
	String analyzerBeansConfigurationDatastores = ConfigurationSerializer
			.serializeAnalyzerBeansConfigurationDataStores(analyzerBeansConfiguration);
	String analysisJobXml = ConfigurationSerializer
			.serializeAnalysisJobToXml(analyzerBeansConfiguration,
					analysisJob);
	HBaseTableMapper hBaseTableMapper = new HBaseTableMapper();
	mapDriver = MapDriver.newMapDriver(hBaseTableMapper);
	mapDriver
			.getConfiguration()
			.set("io.serializations",
					"org.apache.hadoop.hbase.mapreduce.ResultSerialization,"
					+ "org.apache.hadoop.hbase.mapreduce.KeyValueSerialization,"
					+ "org.apache.hadoop.hbase.mapreduce.MutationSerialization,"
					+ "org.apache.hadoop.io.serializer.JavaSerialization,"
					+ "org.apache.hadoop.io.serializer.WritableSerialization");
	mapDriver.getConfiguration().set(
			HBaseTool.ANALYZER_BEANS_CONFIGURATION_DATASTORES_KEY,
			analyzerBeansConfigurationDatastores);
	mapDriver.getConfiguration().set(HBaseTool.ANALYSIS_JOB_XML_KEY,
			analysisJobXml);
}
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入依赖的package包/类
public static void verifyMapReduce(SmartMapper mapper, SmartReducer reducer, Object key, Object input)
    throws Exception
{
  MapDriver mapDriver = new MapDriver();
  mapDriver.setMapper(mapper);
  MapReduceDriver mapReduceDriver = new MapReduceDriver();
  mapReduceDriver.setMapper(mapper);
  Object writableKey = WritableUtils.createWritable(key, mapper.getKeyInType());
  Object writableValue = WritableUtils.createWritable(input, mapper.getValueInType());
  mapDriver.withInput(writableKey, writableValue);
  List results = mapDriver.run();
  Collections.sort(results, PairComparer.INSTANCE);
  mapReduceDriver = new MapReduceDriver<LongWritable, Text, Text, LongWritable, Text, LongWritable>();
  writableKey = WritableUtils.createWritable(key, mapper.getKeyInType());
  writableValue = WritableUtils.createWritable(input, mapper.getValueInType());
  mapReduceDriver.withInput(writableKey, writableValue);
  mapReduceDriver.setMapper(mapper);
  mapReduceDriver.setReducer(reducer);
  List finalResults = mapReduceDriver.run();
  String text = String.format("[%s]\n\n -> maps via %s to -> \n\n%s\n\n -> reduces via %s to -> \n\n%s", input,
      mapper.getClass().getSimpleName(), ArrayUtils.toString(results, Echo.INSTANCE),
      reducer.getClass().getSimpleName(), ArrayUtils.toString(finalResults, Echo.INSTANCE));
  Approvals.verify(text);
}
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入依赖的package包/类
/**
 * Set up the FlintHadoop tests
 * @throws InstantiationException
 * @throws IllegalAccessException
 */
@Before
public void setUp() throws InstantiationException, IllegalAccessException {
    FlintHadoop.FlintMap mapper = new FlintHadoop.FlintMap();
   // FlintHadoop.FlintReduce reducer = new FlintHadoop.FlintReduce();
    mapDriver = MapDriver.newMapDriver(mapper);
    //reduceDriver = ReduceDriver.newReduceDriver(reducer);
    //mapRedDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);

    mapDriver.getConfiguration().set("mapred.output.dir", tmpDir.getAbsolutePath());
    //reduceDriver.getConfiguration().set("mapred.output.dir", tmpDir.getAbsolutePath());
    //mapRedDriver.getConfiguration().set("mapred.output.dir", tmpDir.getAbsolutePath());

    testPdf1CheckResult =  new Flint().check(new File(testPdf1Path)).get(0);
    testPdf2CheckResult =  new Flint().check(new File(testPdf2Path)).get(0);
}