Java DistributedCache.addCacheArchive方法代码示例

import org.apache.hadoop.filecache.DistributedCache; //导入方法依赖的package包/类
public static void addSolrConfToDistributedCache(Job job, File solrHomeZip)
    throws IOException {
  // Make a reasonably unique name for the zip file in the distributed cache
  // to avoid collisions if multiple jobs are running.
  String hdfsZipName = UUID.randomUUID().toString() + '.'
      + ZIP_FILE_BASE_NAME;
  Configuration jobConf = job.getConfiguration();
  jobConf.set(ZIP_NAME, hdfsZipName);

  Path zipPath = new Path("/tmp", getZipName(jobConf));
  FileSystem fs = FileSystem.get(jobConf);
  fs.copyFromLocalFile(new Path(solrHomeZip.toString()), zipPath);
  final URI baseZipUrl = fs.getUri().resolve(
      zipPath.toString() + '#' + getZipName(jobConf));

  DistributedCache.addCacheArchive(baseZipUrl, jobConf);
  LOG.debug("Set Solr distributed cache: {}", Arrays.asList(job.getCacheArchives()));
  LOG.debug("Set zipPath: {}", zipPath);
  // Actually send the path for the configuration zip file
  jobConf.set(SETUP_OK, zipPath.toString());
}
 

import org.apache.hadoop.filecache.DistributedCache; //导入方法依赖的package包/类
public void testDuplicationsMinicluster() throws Exception {
  OutputStream os = getFileSystem().create(new Path(getInputDir(), "text.txt"));
  Writer wr = new OutputStreamWriter(os);
  wr.write("hello1\n");
  wr.write("hello2\n");
  wr.write("hello3\n");
  wr.write("hello4\n");
  wr.close();

  JobConf conf = createJobConf();
  conf.setJobName("counters");
  
  conf.setInputFormat(TextInputFormat.class);

  conf.setMapOutputKeyClass(LongWritable.class);
  conf.setMapOutputValueClass(Text.class);

  conf.setOutputFormat(TextOutputFormat.class);
  conf.setOutputKeyClass(LongWritable.class);
  conf.setOutputValueClass(Text.class);

  conf.setMapperClass(IdentityMapper.class);
  conf.setReducerClass(IdentityReducer.class);

  FileInputFormat.setInputPaths(conf, getInputDir());

  FileOutputFormat.setOutputPath(conf, getOutputDir());

  Path inputRoot = getInputDir().makeQualified(getFileSystem());
  Path unqualifiedInputRoot = getInputDir();
  System.out.println("The qualified input dir is " + inputRoot.toString());
  System.out.println("The unqualified input dir is " + unqualifiedInputRoot.toString());

  Path duplicatedPath = new Path(inputRoot, "text.txt");
  URI duplicatedURI = duplicatedPath.toUri();

  Path unqualifiedDuplicatedPath = new Path(unqualifiedInputRoot, "text.txt");
  URI unqualifiedDuplicatedURI = unqualifiedDuplicatedPath.toUri();

  System.out.println("The duplicated Path is " + duplicatedPath);
  System.out.println("The duplicated URI is " + duplicatedURI);
  System.out.println("The unqualified duplicated URI is " + unqualifiedDuplicatedURI);

  DistributedCache.addCacheArchive(duplicatedURI, conf);
  DistributedCache.addCacheFile(unqualifiedDuplicatedURI, conf);

  try {
    RunningJob runningJob = JobClient.runJob(conf);

    assertFalse("The job completed, which is wrong since there's a duplication", true);
  } catch (InvalidJobConfException e) {
    System.out.println("We expect to see a stack trace here.");
    e.printStackTrace(System.out);
  }
}