Java MapReduceDriver.newMapReduceDriver方法代码示例

import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; //导入方法依赖的package包/类
@Before
public void setUp() {
    AnalyzerBeansConfiguration analyzerBeansConfiguration = buildAnalyzerBeansConfigurationLocalFS(CSV_FILE_PATH);
    analysisJob = buildAnalysisJob(analyzerBeansConfiguration, CSV_FILE_PATH);
    String analyzerBeansConfigurationDatastores = ConfigurationSerializer
            .serializeAnalyzerBeansConfigurationDataStores(analyzerBeansConfiguration);
    String analysisJobXml = ConfigurationSerializer.serializeAnalysisJobToXml(analyzerBeansConfiguration,
            analysisJob);
    FlatFileMapper flatFileMapper = new FlatFileMapper();
    FlatFileReducer flatFileReducer = new FlatFileReducer();
    mapDriver = MapDriver.newMapDriver(flatFileMapper);
    mapDriver.getConfiguration().set(FlatFileTool.ANALYZER_BEANS_CONFIGURATION_DATASTORES_KEY,
            analyzerBeansConfigurationDatastores);
    mapDriver.getConfiguration().set(FlatFileTool.ANALYSIS_JOB_XML_KEY, analysisJobXml);
    reduceDriver = ReduceDriver.newReduceDriver(flatFileReducer);
    reduceDriver.getConfiguration().set(FlatFileTool.ANALYZER_BEANS_CONFIGURATION_DATASTORES_KEY,
            analyzerBeansConfigurationDatastores);
    reduceDriver.getConfiguration().set(FlatFileTool.ANALYSIS_JOB_XML_KEY, analysisJobXml);
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(flatFileMapper, flatFileReducer);
}
 

import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; //导入方法依赖的package包/类
@Before
public void setUp() {
  CopyMapper mapper = new Migration.CopyMapper();
  IdentityReducer reducer = new Migration.IdentityReducer();
  mapDriver = MapDriver.newMapDriver(mapper);
  reduceDriver = ReduceDriver.newReduceDriver(reducer);
  mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
 

import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; //导入方法依赖的package包/类
@Before
public void setUp()
{
    SMSCDRMapper mapper = new SMSCDRMapper();
    SMSCDRReducer reducer = new SMSCDRReducer();
    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
 

import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; //导入方法依赖的package包/类
@Before
public void setup() {
	TestMapper mapper = new TestMapper();
	TestReducer reducer = new TestReducer();
	driver = MapReduceDriver.newMapReduceDriver(mapper, reducer);

	Configuration conf = driver.getConfiguration();
	Job job = mock(Job.class);
	when(job.getConfiguration()).thenReturn(conf);

	CompositeSortKeySerialization.configureMapOutputKey(job, Text.class, IntWritable.class);

	// MRUnit sets these differently than standard MapReduce:
	driver.setKeyGroupingComparator(new CompositeSortKey.GroupingComparator<Text, IntWritable>());
}
 

import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; //导入方法依赖的package包/类
@Before
public void before() {
    PostcodeMapper mapper = new PostcodeMapper();
    PostcodeReducer combiner = new PostcodeReducer();
    PostcodeReducer reducer = new PostcodeReducer();

    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer, combiner);
}
 

import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; //导入方法依赖的package包/类
@Before
public void before() throws URISyntaxException {
    CsvFieldCountMapper mapper = new CsvFieldCountMapper();
    LongSumReducer<Text> combiner = new LongSumReducer<Text>();
    LongSumReducer<Text> reducer = new LongSumReducer<Text>();
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer, combiner);
    Configuration conf = mapReduceDriver.getConfiguration();
    conf.setInt(CsvFieldCountMapper.CSV_FIELD_IDX, 2);
    conf.set(CsvFieldCountMapper.FILTER_CACHE_FILE_NAME, "fr_urban_postcodes.txt");
    mapReduceDriver.addCacheFile(new File("target/test-classes/referential/fr_urban_postcodes.txt").toURI());

}
 

import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; //导入方法依赖的package包/类
@Before
public void before() throws URISyntaxException {
    CsvFieldCountMapper mapper = new CsvFieldCountMapper();
    LongSumReducer<Text> combiner = new LongSumReducer<Text>();
    LongSumReducer<Text> reducer = new LongSumReducer<Text>();
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer, combiner);
    Configuration conf = mapReduceDriver.getConfiguration();
    conf.setInt(CsvFieldCountMapper.CSV_FIELD_IDX, 2);
}
 

import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; //导入方法依赖的package包/类
@Before
public void setUp() throws Exception {
    createTestMetadata();

    // hack for distributed cache
    FileUtils.deleteDirectory(new File("./meta"));
    FileUtils.copyDirectory(new File(getTestConfig().getMetadataUrl().toString()), new File("./meta"));

    NDCuboidMapper mapper = new NDCuboidMapper();
    CuboidReducer reducer = new CuboidReducer();
    mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
 

import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; //导入方法依赖的package包/类
@Before
public void setUp() throws Exception {
	Mapper<Writable,Text, Text, Text> mapper = new IdentityJSONMapper();
	Reducer<Text,Text,Text,Text> reducer = new JSONReducer();

	mapDriver = MapDriver.newMapDriver(mapper);
	reduceDriver = ReduceDriver.newReduceDriver(reducer);
	mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
 

import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; //导入方法依赖的package包/类
@Before
public void setUp() throws Exception {
	Mapper<Writable,Text, Text, Text> mapper = new JSONTypeFilterMapper();
	Reducer<Text,Text,Text,Text> reducer = new JSONReducer();

	mapDriver = MapDriver.newMapDriver(mapper);
	reduceDriver = ReduceDriver.newReduceDriver(reducer);
	mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
 

import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; //导入方法依赖的package包/类
@Before
public void setUp() throws Exception {
	Mapper<Text, Text, Text, Text> mapper = new JSONMapper();
	Reducer<Text,Text,Text,Text> reducer = new JSONReducer();

	mapDriver = MapDriver.newMapDriver(mapper);
	org.apache.hadoop.conf.Configuration conf = mapDriver.getConfiguration();
	conf.set("model.uri", TestJSONMapReduce.class.getClassLoader().getResource("people-model.ttl").toURI().toString());
	conf.set("rdf.generation.root", "http://isi.edu/integration/karma/dev#TriplesMap_c6f9c495-90e4-4c83-aa62-0ab1841a1871");
	reduceDriver = ReduceDriver.newReduceDriver(reducer);
	mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
 

import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; //导入方法依赖的package包/类
@BeforeClass
public static void setUpBeforeClass() throws Exception {
	Mapper<Writable,Text, Text, Text> mapper = new IdentityJSONMapper();
	Reducer<Text,Text,Text,Text> reducer = new JSONReducer();
	mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
	filePath = System.getProperty("json.filepath");
}
 

import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; //导入方法依赖的package包/类
@Before
public void setUp() throws Exception {
	Mapper<Text, Text, Text, Text> mapper = new N3Mapper();
	Reducer<Text,Text,Text,Text> reducer = new N3Reducer();

	mapDriver = MapDriver.newMapDriver(mapper);
	reduceDriver = ReduceDriver.newReduceDriver(reducer);
	mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
 

import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; //导入方法依赖的package包/类
@Before
public void setUp() throws Exception {
	Mapper<Writable,Text, Text, Text> mapper = new IdentityN3Mapper();
	Reducer<Text,Text,Text,Text> reducer = new N3ToJSONReducer();

	mapDriver = MapDriver.newMapDriver(mapper);
	reduceDriver = ReduceDriver.newReduceDriver(reducer);
	mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
}
 

import org.apache.hadoop.mrunit.mapreduce.MapReduceDriver; //导入方法依赖的package包/类
@Before
public void init(){
  ToItemCooccurrenceMapper mapper = new ToItemCooccurrenceMapper();
  ToItemCooccurrenceReducer reducer = new ToItemCooccurrenceReducer();
  mapDriver = MapDriver.newMapDriver(mapper);
  reduceDriver = ReduceDriver.newReduceDriver(reducer);
  mapReduceDriver = MapReduceDriver.newMapReduceDriver(mapper, reducer);
  //mapReduceDriver = new MapReduceDriver<LongWritable, Text,VarLongWritable, Text, VarLongWritable, VectorWritable>(mapper,reducer);
}