Java DistributedCache.addFileToClassPath方法代码示例

import org.apache.hadoop.mapreduce.filecache.DistributedCache; //导入方法依赖的package包/类
static boolean runJob(JobConf conf, Path inDir, Path outDir, int numMaps, 
                         int numReds) throws IOException, InterruptedException {

  FileSystem fs = FileSystem.get(conf);
  if (fs.exists(outDir)) {
    fs.delete(outDir, true);
  }
  if (!fs.exists(inDir)) {
    fs.mkdirs(inDir);
  }
  String input = "The quick brown fox\n" + "has many silly\n"
      + "red fox sox\n";
  for (int i = 0; i < numMaps; ++i) {
    DataOutputStream file = fs.create(new Path(inDir, "part-" + i));
    file.writeBytes(input);
    file.close();
  }

  DistributedCache.addFileToClassPath(TestMRJobs.APP_JAR, conf, fs);
  conf.setOutputCommitter(CustomOutputCommitter.class);
  conf.setInputFormat(TextInputFormat.class);
  conf.setOutputKeyClass(LongWritable.class);
  conf.setOutputValueClass(Text.class);

  FileInputFormat.setInputPaths(conf, inDir);
  FileOutputFormat.setOutputPath(conf, outDir);
  conf.setNumMapTasks(numMaps);
  conf.setNumReduceTasks(numReds);

  JobClient jobClient = new JobClient(conf);
  
  RunningJob job = jobClient.submitJob(conf);
  return jobClient.monitorAndPrintJob(conf, job);
}
 

import org.apache.hadoop.mapreduce.filecache.DistributedCache; //导入方法依赖的package包/类
@Test
public void testCombinerShouldUpdateTheReporter() throws Exception {
  JobConf conf = new JobConf(mrCluster.getConfig());
  int numMaps = 5;
  int numReds = 2;
  Path in = new Path(mrCluster.getTestWorkDir().getAbsolutePath(),
      "testCombinerShouldUpdateTheReporter-in");
  Path out = new Path(mrCluster.getTestWorkDir().getAbsolutePath(),
      "testCombinerShouldUpdateTheReporter-out");
  createInputOutPutFolder(in, out, numMaps);
  conf.setJobName("test-job-with-combiner");
  conf.setMapperClass(IdentityMapper.class);
  conf.setCombinerClass(MyCombinerToCheckReporter.class);
  //conf.setJarByClass(MyCombinerToCheckReporter.class);
  conf.setReducerClass(IdentityReducer.class);
  DistributedCache.addFileToClassPath(TestMRJobs.APP_JAR, conf);
  conf.setOutputCommitter(CustomOutputCommitter.class);
  conf.setInputFormat(TextInputFormat.class);
  conf.setOutputKeyClass(LongWritable.class);
  conf.setOutputValueClass(Text.class);

  FileInputFormat.setInputPaths(conf, in);
  FileOutputFormat.setOutputPath(conf, out);
  conf.setNumMapTasks(numMaps);
  conf.setNumReduceTasks(numReds);
  
  runJob(conf);
}
 

import org.apache.hadoop.mapreduce.filecache.DistributedCache; //导入方法依赖的package包/类
@SuppressWarnings("deprecation")
private void copyLog4jPropertyFile(Job job, Path submitJobDir,
    short replication) throws IOException {
  Configuration conf = job.getConfiguration();

  String file =
      validateFilePath(
          conf.get(MRJobConfig.MAPREDUCE_JOB_LOG4J_PROPERTIES_FILE), conf);
  LOG.debug("default FileSystem: " + jtFs.getUri());
  FsPermission mapredSysPerms =
      new FsPermission(JobSubmissionFiles.JOB_DIR_PERMISSION);
  if (!jtFs.exists(submitJobDir)) {
    throw new IOException("Cannot find job submission directory! "
        + "It should just be created, so something wrong here.");
  }

  Path fileDir = JobSubmissionFiles.getJobLog4jFile(submitJobDir);

  // first copy local log4j.properties file to HDFS under submitJobDir
  if (file != null) {
    FileSystem.mkdirs(jtFs, fileDir, mapredSysPerms);
    URI tmpURI = null;
    try {
      tmpURI = new URI(file);
    } catch (URISyntaxException e) {
      throw new IllegalArgumentException(e);
    }
    Path tmp = new Path(tmpURI);
    Path newPath = copyRemoteFiles(fileDir, tmp, conf, replication);
    DistributedCache.addFileToClassPath(new Path(newPath.toUri().getPath()),
        conf);
  }
}
 

import org.apache.hadoop.mapreduce.filecache.DistributedCache; //导入方法依赖的package包/类
@SuppressWarnings("deprecation")
private void copyLog4jPropertyFile(Job job, Path submitJobDir,
    short replication) throws IOException {
  Configuration conf = job.getConfiguration();

  String file = validateFilePath(
      conf.get(MRJobConfig.MAPREDUCE_JOB_LOG4J_PROPERTIES_FILE), conf);
  LOG.debug("default FileSystem: " + jtFs.getUri());
  FsPermission mapredSysPerms = 
    new FsPermission(JobSubmissionFiles.JOB_DIR_PERMISSION);
  if (!jtFs.exists(submitJobDir)) {
    throw new IOException("Cannot find job submission directory! " 
        + "It should just be created, so something wrong here.");
  }
  
  Path fileDir = JobSubmissionFiles.getJobLog4jFile(submitJobDir);

  // first copy local log4j.properties file to HDFS under submitJobDir
  if (file != null) {
    FileSystem.mkdirs(jtFs, fileDir, mapredSysPerms);
    URI tmpURI = null;
    try {
      tmpURI = new URI(file);
    } catch (URISyntaxException e) {
      throw new IllegalArgumentException(e);
    }
    Path tmp = new Path(tmpURI);
    Path newPath = copyRemoteFiles(fileDir, tmp, conf, replication);
    DistributedCache.addFileToClassPath(new Path(newPath.toUri().getPath()), conf);
  }
}
 

import org.apache.hadoop.mapreduce.filecache.DistributedCache; //导入方法依赖的package包/类
public void addCacheFiles( final String filesString , final Configuration conf ) throws IOException{
  for( String file : filesString.split( "," ) ){
    DistributedCache.addFileToClassPath( new Path( file ) , conf );
  }
}
 

import org.apache.hadoop.mapreduce.filecache.DistributedCache; //导入方法依赖的package包/类
/**
 * Add an file path to the current set of classpath entries It adds the file
 * to cache as well.
 * 
 * Files added with this method will not be unpacked while being added to the
 * classpath.
 * To add archives to classpath, use the {@link #addArchiveToClassPath(Path)}
 * method instead.
 *
 * @param file Path of the file to be added
 */
public void addFileToClassPath(Path file)
  throws IOException {
  ensureState(JobState.DEFINE);
  DistributedCache.addFileToClassPath(file, conf, file.getFileSystem(conf));
}