Java JarFinder类代码示例

import org.apache.hadoop.util.JarFinder; //导入依赖的package包/类
@Test
public void testGoodClassOrNull() throws Exception {
  String NAME = "ClassWithNoPackage";
  ClassLoader cl = TestClassWithNoPackage.class.getClassLoader();
  String JAR = JarFinder.getJar(cl.loadClass(NAME));

  // Add testjob jar file to classpath.
  Configuration conf = new Configuration();
  conf.setClassLoader(new URLClassLoader(new URL[]{new URL("file", null, JAR)}, 
                                         null));
  // Get class with no package name.
  String defaultPackage = this.getClass().getPackage().getName();
  Class c = StreamUtil.goodClassOrNull(conf, NAME, defaultPackage);
  assertNotNull("Class " + NAME + " not found!", c);
}
 

import org.apache.hadoop.util.JarFinder; //导入依赖的package包/类
public static MiniMRClientCluster create(Class<?> caller, String identifier,
    int noOfNMs, Configuration conf) throws IOException {

  if (conf == null) {
    conf = new Configuration();
  }

  FileSystem fs = FileSystem.get(conf);

  Path testRootDir = new Path("target", identifier + "-tmpDir")
      .makeQualified(fs);
  Path appJar = new Path(testRootDir, "MRAppJar.jar");

  // Copy MRAppJar and make it private.
  Path appMasterJar = new Path(MiniMRYarnCluster.APPJAR);

  fs.copyFromLocalFile(appMasterJar, appJar);
  fs.setPermission(appJar, new FsPermission("744"));

  Job job = Job.getInstance(conf);

  job.addFileToClassPath(appJar);

  Path callerJar = new Path(JarFinder.getJar(caller));
  Path remoteCallerJar = new Path(testRootDir, callerJar.getName());
  fs.copyFromLocalFile(callerJar, remoteCallerJar);
  fs.setPermission(remoteCallerJar, new FsPermission("744"));
  job.addFileToClassPath(remoteCallerJar);

  MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(identifier,
      noOfNMs);
  job.getConfiguration().set("minimrclientcluster.caller.name",
      identifier);
  job.getConfiguration().setInt("minimrclientcluster.nodemanagers.number",
      noOfNMs);
  miniMRYarnCluster.init(job.getConfiguration());
  miniMRYarnCluster.start();

  return new MiniMRYarnClusterAdapter(miniMRYarnCluster);
}
 

import org.apache.hadoop.util.JarFinder; //导入依赖的package包/类
public static MiniMRClientCluster create(Class<?> caller, String identifier,
    int noOfNMs, Configuration conf, boolean formatDB) throws IOException {

  if (conf == null) {
    conf = new Configuration();
  }

  FileSystem fs = FileSystem.get(conf);

  Path testRootDir = new Path("target", identifier + "-tmpDir")
      .makeQualified(fs);
  Path appJar = new Path(testRootDir, "MRAppJar.jar");

  // Copy MRAppJar and make it private.
  Path appMasterJar = new Path(MiniMRYarnCluster.APPJAR);

  fs.copyFromLocalFile(appMasterJar, appJar);
  fs.setPermission(appJar, new FsPermission("744"));

  Job job = Job.getInstance(conf);

  job.addFileToClassPath(appJar);

  Path callerJar = new Path(JarFinder.getJar(caller));
  Path remoteCallerJar = new Path(testRootDir, callerJar.getName());
  fs.copyFromLocalFile(callerJar, remoteCallerJar);
  fs.setPermission(remoteCallerJar, new FsPermission("744"));
  job.addFileToClassPath(remoteCallerJar);

  MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(identifier,
      noOfNMs, formatDB);
  job.getConfiguration().set("minimrclientcluster.caller.name",
      identifier);
  job.getConfiguration().setInt("minimrclientcluster.nodemanagers.number",
      noOfNMs);
  miniMRYarnCluster.init(job.getConfiguration());
  miniMRYarnCluster.start();

  return new MiniMRYarnClusterAdapter(miniMRYarnCluster);
}
 

import org.apache.hadoop.util.JarFinder; //导入依赖的package包/类
public void _testDistributedCache(String jobJarPath) throws Exception {
  if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
    LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR
         + " not found. Not running test.");
    return;
  }

  // Create a temporary file of length 1.
  Path first = createTempFile("distributed.first", "x");
  // Create two jars with a single file inside them.
  Path second =
      makeJar(new Path(TEST_ROOT_DIR, "distributed.second.jar"), 2);
  Path third =
      makeJar(new Path(TEST_ROOT_DIR, "distributed.third.jar"), 3);
  Path fourth =
      makeJar(new Path(TEST_ROOT_DIR, "distributed.fourth.jar"), 4);

  Job job = Job.getInstance(mrCluster.getConfig());
  
  // Set the job jar to a new "dummy" jar so we can check that its extracted 
  // properly
  job.setJar(jobJarPath);
  // Because the job jar is a "dummy" jar, we need to include the jar with
  // DistributedCacheChecker or it won't be able to find it
  Path distributedCacheCheckerJar = new Path(
          JarFinder.getJar(DistributedCacheChecker.class));
  job.addFileToClassPath(distributedCacheCheckerJar.makeQualified(
          localFs.getUri(), distributedCacheCheckerJar.getParent()));
  
  job.setMapperClass(DistributedCacheChecker.class);
  job.setOutputFormatClass(NullOutputFormat.class);

  FileInputFormat.setInputPaths(job, first);
  // Creates the Job Configuration
  job.addCacheFile(
      new URI(first.toUri().toString() + "#distributed.first.symlink"));
  job.addFileToClassPath(second);
  // The AppMaster jar itself
  job.addFileToClassPath(
          APP_JAR.makeQualified(localFs.getUri(), APP_JAR.getParent())); 
  job.addArchiveToClassPath(third);
  job.addCacheArchive(fourth.toUri());
  job.setMaxMapAttempts(1); // speed up failures

  job.submit();
  String trackingUrl = job.getTrackingURL();
  String jobId = job.getJobID().toString();
  Assert.assertTrue(job.waitForCompletion(false));
  Assert.assertTrue("Tracking URL was " + trackingUrl +
                    " but didn't Match Job ID " + jobId ,
        trackingUrl.endsWith(jobId.substring(jobId.lastIndexOf("_")) + "/"));
}
 

import org.apache.hadoop.util.JarFinder; //导入依赖的package包/类
public static MiniMRClientCluster create(Class<?> caller, String identifier,
    int noOfNMs, Configuration conf, File testWorkDir) throws IOException {

  if (conf == null) {
    conf = new Configuration();
  }

  FileSystem fs = FileSystem.get(conf);

  Path testRootDir = new Path(testWorkDir.getPath(), identifier + "-tmpDir")
      .makeQualified(fs);
  Path appJar = new Path(testRootDir, "MRAppJar.jar");

  // Copy MRAppJar and make it private.
  Path appMasterJar = new Path(MiniMRYarnCluster.APPJAR);

  fs.copyFromLocalFile(appMasterJar, appJar);
  fs.setPermission(appJar, new FsPermission("744"));

  Job job = Job.getInstance(conf);

  job.addFileToClassPath(appJar);

  Path callerJar = new Path(JarFinder.getJar(caller));
  Path remoteCallerJar = new Path(testRootDir, callerJar.getName());
  fs.copyFromLocalFile(callerJar, remoteCallerJar);
  fs.setPermission(remoteCallerJar, new FsPermission("744"));
  job.addFileToClassPath(remoteCallerJar);

  MiniMRYarnCluster miniMRYarnCluster;
  try {
    miniMRYarnCluster = new MiniMRYarnCluster(identifier,
        noOfNMs, testWorkDir);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
  job.getConfiguration().set("minimrclientcluster.caller.name",
      identifier);
  job.getConfiguration().setInt("minimrclientcluster.nodemanagers.number",
      noOfNMs);
  miniMRYarnCluster.init(job.getConfiguration());
  miniMRYarnCluster.start();

  return new MiniMRYarnClusterAdapter(miniMRYarnCluster, testWorkDir);
}
 

import org.apache.hadoop.util.JarFinder; //导入依赖的package包/类
public YarnComponent withApplicationMasterClass(Class clazz){
   appmasterJar = JarFinder.getJar(clazz);
    return this;
}