Java MapDriver.run方法代码示例

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入方法依赖的package包/类
@Test
public void testMapper() throws Exception {
  MorphlineMapper mapper = new MorphlineMapper();
  MapDriver<LongWritable, Text, Text, SolrInputDocumentWritable> mapDriver = MapDriver.newMapDriver(mapper);;

  Configuration config = mapDriver.getConfiguration();
  setupHadoopConfig(config);

  mapDriver.withInput(new LongWritable(0L), new Text("hdfs://localhost/" + DOCUMENTS_DIR + "/sample-statuses-20120906-141433.avro"));

  SolrInputDocument sid = new SolrInputDocument();
  sid.addField("id", "uniqueid1");
  sid.addField("user_name", "user1");
  sid.addField("text", "content of record one");
  SolrInputDocumentWritable sidw = new SolrInputDocumentWritable(sid);

  mapDriver
    .withCacheArchive(solrHomeZip.getAbsolutePath())
    .withOutput(new Text("0"), sidw);
  //mapDriver.runTest();
  List<Pair<Text, SolrInputDocumentWritable>> result = mapDriver.run();
  for (Pair<Text, SolrInputDocumentWritable> p: result) {
    System.out.println(p.getFirst());
    System.out.println(p.getSecond());
  }
}
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入方法依赖的package包/类
public static void verifyMapReduce(SmartMapper mapper, SmartReducer reducer, Object key, Object input)
    throws Exception
{
  MapDriver mapDriver = new MapDriver();
  mapDriver.setMapper(mapper);
  MapReduceDriver mapReduceDriver = new MapReduceDriver();
  mapReduceDriver.setMapper(mapper);
  Object writableKey = WritableUtils.createWritable(key, mapper.getKeyInType());
  Object writableValue = WritableUtils.createWritable(input, mapper.getValueInType());
  mapDriver.withInput(writableKey, writableValue);
  List results = mapDriver.run();
  Collections.sort(results, PairComparer.INSTANCE);
  mapReduceDriver = new MapReduceDriver<LongWritable, Text, Text, LongWritable, Text, LongWritable>();
  writableKey = WritableUtils.createWritable(key, mapper.getKeyInType());
  writableValue = WritableUtils.createWritable(input, mapper.getValueInType());
  mapReduceDriver.withInput(writableKey, writableValue);
  mapReduceDriver.setMapper(mapper);
  mapReduceDriver.setReducer(reducer);
  List finalResults = mapReduceDriver.run();
  String text = String.format("[%s]\n\n -> maps via %s to -> \n\n%s\n\n -> reduces via %s to -> \n\n%s", input,
      mapper.getClass().getSimpleName(), ArrayUtils.toString(results, Echo.INSTANCE),
      reducer.getClass().getSimpleName(), ArrayUtils.toString(finalResults, Echo.INSTANCE));
  Approvals.verify(text);
}
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入方法依赖的package包/类
private void generateAndSerializeStarTree() throws IOException
{
  List<Pair<BytesWritable,  BytesWritable>> inputRecords = new ArrayList<Pair<BytesWritable,  BytesWritable>>();
  String []combination1 = {"A1", "B1", "C1"};
  DimensionKey dimKey = new DimensionKey(combination1);
  MetricTimeSeries timeSeries = TestHelper.generateRandomMetricTimeSeries(starTreeBootstrapConfig);
  Pair<BytesWritable,  BytesWritable> record = new Pair<BytesWritable, BytesWritable>(new BytesWritable(dimKey.toBytes())
                                                                        , new BytesWritable(timeSeries.toBytes()));
  inputRecords.add(record);

  String []combination2 = {"A1", "B1", "C2"};
  dimKey = new DimensionKey(combination2);
  timeSeries = TestStarTreeBootstrapPhase2.TestHelper.generateRandomMetricTimeSeries(starTreeBootstrapConfig);
  record = new Pair<BytesWritable, BytesWritable>(new BytesWritable(dimKey.toBytes())
                                                                        , new BytesWritable(timeSeries.toBytes()));
  inputRecords.add(record);

  String []combination3 = {"A1", "B1", "C3"};
  dimKey = new DimensionKey(combination3);
  timeSeries = TestStarTreeBootstrapPhase2.TestHelper.generateRandomMetricTimeSeries(starTreeBootstrapConfig);
  record = new Pair<BytesWritable, BytesWritable>(new BytesWritable(dimKey.toBytes())
                                                                        , new BytesWritable(timeSeries.toBytes()));
  inputRecords.add(record);

  String []combination4 = {"A2", "B1", "C3"};
  dimKey = new DimensionKey(combination4);
  timeSeries = TestStarTreeBootstrapPhase2.TestHelper.generateRandomMetricTimeSeries(starTreeBootstrapConfig);
  record = new Pair<BytesWritable, BytesWritable>(new BytesWritable(dimKey.toBytes())
                                                                        , new BytesWritable(timeSeries.toBytes()));
  inputRecords.add(record);

  MapDriver<BytesWritable, BytesWritable, BytesWritable, BytesWritable> mapDriver;
  StarTreeGenerationMapper mapper = new StarTreeGenerationMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration config = mapDriver.getConfiguration();
  config.set(StarTreeGenerationConstants.STAR_TREE_GEN_CONFIG_PATH.toString(), ClassLoader.getSystemResource(CONF_FILE).toString());
  config.set(StarTreeGenerationConstants.STAR_TREE_GEN_OUTPUT_PATH.toString(), thirdEyeRoot + File.separator + "startree_generation");
  mapDriver.addAll(inputRecords);
  mapDriver.run();
}
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入方法依赖的package包/类
/**
 * Tests that an error thrown from JavaScript works as expected.
 *
 * @throws Exception if anything goes wrong
 */
@Test
public void testThrowingErrorsWorks() throws Exception {
    final MapDriver<WritableComparable<?>, Writable, WritableComparable<?>, Writable> driver =
            new MapDriver<>();
    final LembosMapper mapper = new LembosMapper();
    final String moduleName = "LembosMapReduceEnvironmentTest-testThrowingErrorsWorks";

    driver.withMapper(mapper);

    driver.getConfiguration().set(LembosConstants.MR_MODULE_NAME, moduleName);
    driver.getConfiguration().set(LembosConstants.MR_MODULE_PATH, TestUtils.getModulePath(moduleName));

    final List<Pair<WritableComparable<?>, Writable>> inputs = ImmutableList.of(
            new Pair<WritableComparable<?>, Writable>(new Text(Long.toString(new Date().getTime())),
                                                             new Text("Alice")),
            new Pair<WritableComparable<?>, Writable>(new Text(Long.toString(new Date().getTime())),
                                                             new Text("Bob")),
            new Pair<WritableComparable<?>, Writable>(new Text(Long.toString(new Date().getTime())),
                                                             new Text("Sally")),
            new Pair<WritableComparable<?>, Writable>(new Text(Long.toString(new Date().getTime())),
                                                             new Text("Bob")),
            new Pair<WritableComparable<?>, Writable>(new Text(Long.toString(new Date().getTime())),
                                                             new Text("Alice"))
    );

    driver.withAll(inputs);

    try {
        driver.run();

        fail("The line above should had failed");
    } catch (RuntimeException e) {
        assertEquals("Error: This is an Error!", e.getMessage());
    }
}
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入方法依赖的package包/类
/**
 * Tests using a Node.js module works as expected.
 *
 * @throws Exception if anything goes wrong
 */
@Test
public void testUsingNodeJSModule() throws Exception {
    final MapDriver<WritableComparable<?>, Writable, WritableComparable<?>, Writable> driver =
            new MapDriver<>();
    final LembosMapper mapper = new LembosMapper();
    final String moduleName = "LembosMapReduceEnvironmentTest-testUsingNodeJSModule";

    driver.withMapper(mapper);

    driver.getConfiguration().set(LembosConstants.MR_MODULE_NAME, moduleName);
    driver.getConfiguration().set(LembosConstants.MR_MODULE_PATH, TestUtils.getModulePath(moduleName));

    final List<Pair<WritableComparable<?>, Writable>> inputs = ImmutableList.of(
            new Pair<WritableComparable<?>, Writable>(new Text(Long.toString(new Date().getTime())),
                                                             new Text("Alice"))
    );

    driver.withAll(inputs);

    final List<Pair<WritableComparable<?>, Writable>> outputs = driver.run();

    assertEquals(1, outputs.size());

    final Pair<WritableComparable<?>, Writable> output = outputs.get(0);
    final String key = output.getFirst().toString();
    final String val = output.getSecond().toString();

    assertEquals("ip", key);
    assertTrue(InetAddressValidator.getInstance().isValidInet4Address(val));
}
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入方法依赖的package包/类
public static void verifyMapping(SmartMapper mapper, Object key, Object input) throws Exception
{
  MapDriver mapDriver = new MapDriver();
  mapDriver.setMapper(mapper);
  Object writableKey = WritableUtils.createWritable(key, mapper.getKeyInType());
  Object writableValue = WritableUtils.createWritable(input, mapper.getValueInType());
  mapDriver.withInput(writableKey, writableValue);
  List results = mapDriver.run();
  Collections.sort(results, PairComparer.INSTANCE);
  String header = String.format("[%s]\n\n -> maps via %s to -> \n", input, mapper.getClass().getSimpleName());
  Approvals.verifyAll(header, results, Echo.INSTANCE);
}
 

import org.apache.hadoop.mrunit.mapreduce.MapDriver; //导入方法依赖的package包/类
private void generateAndSerializeStarTree() throws IOException
{
  List<Pair<BytesWritable,  BytesWritable>> inputRecords = new ArrayList<Pair<BytesWritable,  BytesWritable>>();
  String []combination1 = {"A1", "B1", "C1"};
  DimensionKey dimKey = new DimensionKey(combination1);
  MetricTimeSeries timeSeries = TestStarTreeBootstrapPhase1.TestHelper.generateRandomMetricTimeSeries(starTreeBootstrapConfig);
  Pair<BytesWritable,  BytesWritable> record = new Pair<BytesWritable, BytesWritable>(new BytesWritable(dimKey.toBytes())
                                                                        , new BytesWritable(timeSeries.toBytes()));
  inputRecords.add(record);

  String []combination2 = {"A1", "B1", "C2"};
  dimKey = new DimensionKey(combination2);
  timeSeries = TestStarTreeBootstrapPhase1.TestHelper.generateRandomMetricTimeSeries(starTreeBootstrapConfig);
  record = new Pair<BytesWritable, BytesWritable>(new BytesWritable(dimKey.toBytes())
                                                                        , new BytesWritable(timeSeries.toBytes()));
  inputRecords.add(record);

  String []combination3 = {"A1", "B1", "C3"};
  dimKey = new DimensionKey(combination3);
  timeSeries = TestStarTreeBootstrapPhase1.TestHelper.generateRandomMetricTimeSeries(starTreeBootstrapConfig);
  record = new Pair<BytesWritable, BytesWritable>(new BytesWritable(dimKey.toBytes())
                                                                        , new BytesWritable(timeSeries.toBytes()));
  inputRecords.add(record);

  String []combination4 = {"A2", "B1", "C3"};
  dimKey = new DimensionKey(combination4);
  timeSeries = TestStarTreeBootstrapPhase1.TestHelper.generateRandomMetricTimeSeries(starTreeBootstrapConfig);
  record = new Pair<BytesWritable, BytesWritable>(new BytesWritable(dimKey.toBytes())
                                                                        , new BytesWritable(timeSeries.toBytes()));
  inputRecords.add(record);

  MapDriver<BytesWritable, BytesWritable, BytesWritable, BytesWritable> mapDriver;
  StarTreeGenerationMapper mapper = new StarTreeGenerationMapper();
  mapDriver = MapDriver.newMapDriver(mapper);
  Configuration config = mapDriver.getConfiguration();
  config.set(StarTreeGenerationConstants.STAR_TREE_GEN_CONFIG_PATH.toString(), ClassLoader.getSystemResource(CONF_FILE).toString());
  config.set(StarTreeGenerationConstants.STAR_TREE_GEN_OUTPUT_PATH.toString(), thirdEyeRoot + File.separator + "startree_generation");
  mapDriver.addAll(inputRecords);
  mapDriver.run();

}