Java RunningJob.waitForCompletion方法代码示例

import org.apache.hadoop.mapred.RunningJob; //导入方法依赖的package包/类
private void mrRun() throws Exception {
  FileSystem fs = FileSystem.get(getJobConf());
  Path inputDir = new Path("input");
  fs.mkdirs(inputDir);
  Writer writer = new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
  writer.write("hello");
  writer.close();

  Path outputDir = new Path("output", "output");

  JobConf jobConf = new JobConf(getJobConf());
  jobConf.setInt("mapred.map.tasks", 1);
  jobConf.setInt("mapred.map.max.attempts", 1);
  jobConf.setInt("mapred.reduce.max.attempts", 1);
  jobConf.set("mapred.input.dir", inputDir.toString());
  jobConf.set("mapred.output.dir", outputDir.toString());

  JobClient jobClient = new JobClient(jobConf);
  RunningJob runJob = jobClient.submitJob(jobConf);
  runJob.waitForCompletion();
  assertTrue(runJob.isComplete());
  assertTrue(runJob.isSuccessful());
}
 

import org.apache.hadoop.mapred.RunningJob; //导入方法依赖的package包/类
@Override
protected void runJob(String jobName, Configuration c, List<Scan> scans)
    throws IOException, InterruptedException, ClassNotFoundException {
  JobConf job = new JobConf(TEST_UTIL.getConfiguration());

  job.setJobName(jobName);
  job.setMapperClass(Mapper.class);
  job.setReducerClass(Reducer.class);

  TableMapReduceUtil.initMultiTableSnapshotMapperJob(getSnapshotScanMapping(scans), Mapper.class,
      ImmutableBytesWritable.class, ImmutableBytesWritable.class, job, true, restoreDir);

  TableMapReduceUtil.addDependencyJars(job);

  job.setReducerClass(Reducer.class);
  job.setNumReduceTasks(1); // one to get final "first" and "last" key
  FileOutputFormat.setOutputPath(job, new Path(job.getJobName()));
  LOG.info("Started " + job.getJobName());

  RunningJob runningJob = JobClient.runJob(job);
  runningJob.waitForCompletion();
  assertTrue(runningJob.isSuccessful());
  LOG.info("After map/reduce completion - job " + jobName);
}
 

import org.apache.hadoop.mapred.RunningJob; //导入方法依赖的package包/类
private void encryptedShuffleWithCerts(boolean useClientCerts)
  throws Exception {
  try {
    Configuration conf = new Configuration();
    String keystoresDir = new File(BASEDIR).getAbsolutePath();
    String sslConfsDir =
      KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class);
    KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfsDir, conf,
                                    useClientCerts);
    conf.setBoolean(MRConfig.SHUFFLE_SSL_ENABLED_KEY, true);
    startCluster(conf);
    FileSystem fs = FileSystem.get(getJobConf());
    Path inputDir = new Path("input");
    fs.mkdirs(inputDir);
    Writer writer =
      new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
    writer.write("hello");
    writer.close();

    Path outputDir = new Path("output", "output");

    JobConf jobConf = new JobConf(getJobConf());
    jobConf.setInt("mapred.map.tasks", 1);
    jobConf.setInt("mapred.map.max.attempts", 1);
    jobConf.setInt("mapred.reduce.max.attempts", 1);
    jobConf.set("mapred.input.dir", inputDir.toString());
    jobConf.set("mapred.output.dir", outputDir.toString());
    JobClient jobClient = new JobClient(jobConf);
    RunningJob runJob = jobClient.submitJob(jobConf);
    runJob.waitForCompletion();
    Assert.assertTrue(runJob.isComplete());
    Assert.assertTrue(runJob.isSuccessful());
  } finally {
    stopCluster();
  }
}
 

import org.apache.hadoop.mapred.RunningJob; //导入方法依赖的package包/类
private void encryptedShuffleWithCerts(boolean useClientCerts)
  throws Exception {
  try {
    Configuration conf = new Configuration();
    String keystoresDir = new File(BASEDIR).getAbsolutePath();
    String sslConfsDir =
      KeyStoreTestUtil.getClasspathDir(TestEncryptedShuffle.class);
    KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfsDir, conf,
                                    useClientCerts);
    conf.setBoolean(JobTracker.SHUFFLE_SSL_ENABLED_KEY, true);
    startCluster(conf);
    FileSystem fs = FileSystem.get(getJobConf(conf));
    Path inputDir = new Path("input");
    fs.mkdirs(inputDir);
    Writer writer =
      new OutputStreamWriter(fs.create(new Path(inputDir, "data.txt")));
    writer.write("hello");
    writer.close();

    Path outputDir = new Path("output", "output");

    JobConf jobConf = new JobConf(getJobConf(conf));
    jobConf.setInt("mapred.map.tasks", 1);
    jobConf.setInt("mapred.map.max.attempts", 1);
    jobConf.setInt("mapred.reduce.max.attempts", 1);
    jobConf.set("mapred.input.dir", inputDir.toString());
    jobConf.set("mapred.output.dir", outputDir.toString());
    JobClient jobClient = new JobClient(jobConf);
    RunningJob runJob = jobClient.submitJob(jobConf);
    runJob.waitForCompletion();
    Assert.assertTrue(runJob.isComplete());
    Assert.assertTrue(runJob.isSuccessful());
  } finally {
    stopCluster();
  }
}
 

import org.apache.hadoop.mapred.RunningJob; //导入方法依赖的package包/类
public void start(String[] args, int startIndex) throws IOException {
  try {
    while(startIndex < args.length) {
      String cmd = args[startIndex ++];
      if (cmd.equals(NUM_MAPPERS_KEY)) {
        numMappers = Integer.valueOf(args[startIndex ++]);
      } else if (cmd.equals(NUM_SUBDIRS_KEY)) {
        numSubDirs = Integer.valueOf(args[startIndex ++]);
      } else if (cmd.equals(NUM_FILES_SUB_DIR_KEY)) {
        numFilesEachSubDirs = Integer.valueOf(args[startIndex ++]);
      } else if (cmd.equals(NUM_ROUND_KEY)) {
        round = Integer.valueOf(args[startIndex ++]);
      } else if (cmd.equals(WORKPLACE_KEY)) {
        workplace = args[startIndex ++];
      }
    }
  } catch (Exception e) {
    printUsage();
    System.exit(-1);
  }
  
  if (workplace.trim().isEmpty()) {
    printUsage();
    System.exit(-1);
  }
  
  if (!workplace.endsWith(Path.SEPARATOR)) {
    workplace += Path.SEPARATOR;
  }
  
  JobConf jobConf = createJobConf(conf);
  
  JobClient client = new JobClient(jobConf);
  RunningJob runningJob = client.submitJob(jobConf);
  runningJob.waitForCompletion();
}
 

import org.apache.hadoop.mapred.RunningJob; //导入方法依赖的package包/类
/**
 * This method creates a sample MR job and submits that JobConf object to the
 * static MiniClusterController method to be executed.
 */
@Test
public void testRunningJobLocally() throws IOException, InterruptedException {
  JobConf sampleJob = createWordCountMRJobConf();
  RunningJob runningJob = miniCluster_.runJob(sampleJob);
  runningJob.waitForCompletion();
  assertTrue(runningJob.isSuccessful());
}
 

import org.apache.hadoop.mapred.RunningJob; //导入方法依赖的package包/类
/**
 * This method gets a JobConf object from the static MiniClusterController
 * method, fills it with a sample MR job and then executes the job.
 */
@Test
public void testGetConfigForMiniCluster() throws IOException {
  JobConf sampleJob = miniCluster_.getJobConf(WordCount.class);
  fillInWordCountMRJobConf(sampleJob);
  RunningJob runningJob = JobClient.runJob(sampleJob);
  runningJob.waitForCompletion();
  assertTrue(runningJob.isSuccessful());
}
 

import org.apache.hadoop.mapred.RunningJob; //导入方法依赖的package包/类
public void start(String[] args, int startIndex) throws IOException {
  try {
    while(startIndex < args.length) {
      String cmd = args[startIndex ++];
      if (cmd.equals(NUM_MAPPERS_KEY)) {
        numMappers = Integer.valueOf(args[startIndex ++]);
      } else if (cmd.equals(NUM_SUBDIRS_KEY)) {
        numSubDirs = Integer.valueOf(args[startIndex ++]);
      } else if (cmd.equals(NUM_FILES_SUB_DIR_KEY)) {
        numFilesEachSubDirs = Integer.valueOf(args[startIndex ++]);
      } else if (cmd.equals(NUM_ROUND_KEY)) {
        round = Integer.valueOf(args[startIndex ++]);
      } else if (cmd.equals(WORKPLACE_KEY)) {
        workplace = args[startIndex ++];
      } else if (cmd.equals(NOTIFIER_SERVER_ADDR_KEY)) {
        notifierServerAddrStr = args[startIndex ++];
      } else if (cmd.equals(NOTIFIER_SERVER_PORT_KEY)) {
        notifierServerPortStr = args[startIndex ++];
      } else {
        printUsage();
        System.exit(-1);
      }
    }
  } catch (Exception e) {
    printUsage();
    System.exit(-1);
  }
  
  if (workplace.trim().isEmpty() ||
      notifierServerAddrStr.trim().isEmpty()) {
    printUsage();
    System.exit(-1);
  }
  
  if (!workplace.endsWith(Path.SEPARATOR)) {
    workplace += Path.SEPARATOR;
  }
  
  JobConf jobConf = createJobConf(conf);
  
  JobClient client = new JobClient(jobConf);
  RunningJob runningJob = client.submitJob(jobConf);
  runningJob.waitForCompletion();
}