Java SleepJob.setConf方法代码示例

import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
@Override
public void doAnAction() throws Exception {
  System.out.println("==============================\n" +
      "Submitting job\n" +
      "==================================");
  SleepJob job = new SleepJob();
  job.setConf(conf);
  JobConf jobConf = job.setupJobConf(1, 0, 1, 1, 1, 1);
  JobClient jc = new JobClient(jobConf);
  RunningJob rj;
  try {
    rj = jc.submitJob(jobConf);
  } catch (IOException e) {
    // Job submission is not idempotent, so ignore a failure
    System.out.println("==============================\n" +
        "Job submission failed. Ignore.\n" +
        "==================================");
    return;
  }
  System.out.println("==============================\n" +
      "Successfully submitted job " + rj.getJobID() + "\n" +
      "==================================");
  if (!jc.monitorAndPrintJob(jobConf, rj)) {
    throw new IOException("Job failed! " + rj.getFailureInfo());
  }
}
 

import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
/**
 * Verifying the job summary information for killed job.
 */
@Test
public void testJobSummaryInfoOfKilledJob() throws IOException, 
        InterruptedException {
  SleepJob job = new SleepJob();
  conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false);
  job.setConf(conf);
  conf = job.setupJobConf(2, 1, 4000, 4000, 100, 100);
  JobConf jobConf = new JobConf(conf);
  RunningJob runJob = jobClient.submitJob(jobConf);
  JobID jobId = runJob.getID();
  Assert.assertTrue("Job has not been started for 1 min.", 
          jtClient.isJobStarted(jobId));
  jobClient.killJob(jobId);
  LOG.info("Waiting till the job is completed...");
  Assert.assertTrue("Job has not been completed for 1 min.", 
      jtClient.isJobStopped(jobId));
  JobInfo jInfo = remoteJTClient.getJobInfo(jobId);
  Assert.assertEquals("Job has not been succeeded", 
      jInfo.getStatus().getRunState(), JobStatus.KILLED);
  verifyJobSummaryInfo(jInfo,jobId);
}
 

import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
public JobID runSleepJob(boolean signalJob) throws Exception{
  SleepJob job = new SleepJob();
  job.setConf(conf);
  conf = job.setupJobConf(5, 1, 100, 5, 100, 5);
  JobConf jconf = new JobConf(conf);
  //Controls the job till all verification is done 
  FinishTaskControlAction.configureControlActionForJob(conf);
  //Submitting the job
  RunningJob rJob = cluster.getJTClient().getClient().submitJob(jconf);
  JobID jobId = rJob.getID();
  JobInfo jInfo = remoteJTClient.getJobInfo(jobId);
  LOG.info("jInfo is :" + jInfo);
  boolean jobStarted = cluster.getJTClient().isJobStarted(jobId);
  Assert.assertTrue("Job has not started even after a minute", 
      jobStarted );
    
  if(signalJob) {
    cluster.signalAllTasks(jobId);
    Assert.assertTrue("Job has not stopped yet",
        cluster.getJTClient().isJobStopped(jobId));
  }
  return jobId;
}
 

import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
private void setupJobAndRun() throws IOException { 
  SleepJob job = new SleepJob();
  job.setConf(conf);
  conf = job.setupJobConf(3, 1, 60000, 100, 60000, 100);
  JobConf jobConf = new JobConf(conf);
  cleanup(outputDir, conf);
  jtClient = cluster.getJTClient();
  JobClient client = jtClient.getClient();
  wovenClient = cluster.getJTClient().getProxy();
  RunningJob runJob = client.submitJob(jobConf);
  jID = runJob.getID();
  jInfo = wovenClient.getJobInfo(jID);
  Assert.assertNotNull("Job information is null",jInfo);
  Assert.assertTrue("Job has not been started for 1 min.", 
      jtClient.isJobStarted(jID));
  JobStatus jobStatus = jInfo.getStatus();
  // Make sure that job should run and completes 40%. 
  while (jobStatus.getRunState() != JobStatus.RUNNING && 
    jobStatus.mapProgress() < 0.4f) {
    UtilsForTests.waitFor(100);
    jobStatus = wovenClient.getJobInfo(jID).getStatus();
  }
}
 

import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
RunningJob submitSleepJob(final int numMappers, final int numReducers, 
    final long mapSleepTime,
    final long reduceSleepTime, final boolean shouldComplete, String userInfo,
                                  String queueName) 
                                    throws IOException, InterruptedException {
  JobConf clientConf = new JobConf();
  clientConf.set("mapred.job.tracker", "localhost:"
      + miniMRCluster.getJobTrackerPort());
  UserGroupInformation ugi;
  SleepJob job = new SleepJob();
  job.setConf(clientConf);
  clientConf = job.setupJobConf(numMappers, numReducers, 
      mapSleepTime, (int)mapSleepTime/100,
      reduceSleepTime, (int)reduceSleepTime/100);
  if (queueName != null) {
    clientConf.setQueueName(queueName);
  }
  final JobConf jc = new JobConf(clientConf);
  if (userInfo != null) {
    String[] splits = userInfo.split(",");
    String[] groups = new String[splits.length - 1];
    System.arraycopy(splits, 1, groups, 0, splits.length - 1);
    ugi = UserGroupInformation.createUserForTesting(splits[0], groups);
  } else {
    ugi = UserGroupInformation.getCurrentUser();
  }
  RunningJob rJob = ugi.doAs(new PrivilegedExceptionAction<RunningJob>() {
    public RunningJob run() throws IOException {
      if (shouldComplete) {
        return JobClient.runJob(jc);  
      } else {
        // Job should be submitted as 'userInfo'. So both the client as well as
        // the configuration should point to the same UGI.
        return new JobClient(jc).submitJob(jc);
      }
    }
  });
  return rJob;
}
 

import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
/**
 * Submit the job in different queue and verifying 
 * the job queue information in job summary 
 * after job is completed.
 */
@Test
public void testJobQueueInfoInJobSummary() throws IOException, 
InterruptedException {
  SleepJob job = new SleepJob();
  job.setConf(conf);
  conf = job.setupJobConf(2, 1, 4000, 4000, 100, 100);
  conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", false);
  JobConf jobConf = new JobConf(conf);
  JobQueueInfo [] queues = jobClient.getQueues();
  for (JobQueueInfo queueInfo : queues ){
    if (!queueInfo.getQueueName().equals("default")) {
      queueName = queueInfo.getQueueName();
      break;
    }
  }
  Assert.assertNotNull("No multiple queues in the cluster.",queueName);
  LOG.info("queueName:" + queueName);
  jobConf.setQueueName(queueName);
  RunningJob runJob = jobClient.submitJob(jobConf);
  JobID jobId = runJob.getID();    
  Assert.assertTrue("Job has not been started for 1 min.", 
      jtClient.isJobStarted(jobId));
  LOG.info("Waiting till the job is completed...");
  Assert.assertTrue("Job has not been completed for 1 min.",
      jtClient.isJobStopped(jobId));
  JobInfo jInfo = remoteJTClient.getJobInfo(jobId);
  Assert.assertEquals("Job has not been succeeded", 
      jInfo.getStatus().getRunState(), JobStatus.SUCCEEDED);
  verifyJobSummaryInfo(jInfo,jobId);
}
 

import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
public SleepJobRunnerThread(Configuration conf, int nMappers,
        int nReducers, int sleepTime) {
  super();
  jobToRun = new SleepJob();
  jobToRun.setConf(conf);
  this.nMappers = nMappers;
  this.nReducers = nReducers;
  this.sleepTime = sleepTime;
}
 

import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
/**
 * Verifying the running job status whether it succeeds or not
 * after failing some of its tasks.
 */
@Test
public void testFailedTaskJobStatus() throws IOException, 
        InterruptedException {
  conf = remoteJTClient.getDaemonConf();
  TaskInfo taskInfo = null;
  SleepJob job = new SleepJob();
  job.setConf(conf);
  JobConf jobConf = job.setupJobConf(1, 1, 10000, 4000, 100, 100);
  RunningJob runJob = jobClient.submitJob(jobConf);
  JobID jobId = runJob.getID();
  JobInfo jInfo = remoteJTClient.getJobInfo(jobId);
  Assert.assertTrue("Job has not been started for 1 min.", 
      jtClient.isJobStarted(jobId));
  TaskInfo[] taskInfos = remoteJTClient.getTaskInfo(jobId);
  for (TaskInfo taskinfo : taskInfos) {
    if (!taskinfo.isSetupOrCleanup() && taskinfo.getTaskID().isMap()) {
      taskInfo = taskinfo;
      break;
    }
  }
  Assert.assertTrue("Task has not been started for 1 min.", 
      jtClient.isTaskStarted(taskInfo));

  // Fail the running task.
  NetworkedJob networkJob = jobClient.new NetworkedJob(jInfo.getStatus());
  TaskID tID = TaskID.downgrade(taskInfo.getTaskID());
  TaskAttemptID taskAttID = new TaskAttemptID(tID , 0);
  networkJob.killTask(taskAttID, true);

  LOG.info("Waiting till the job is completed...");
  while (!jInfo.getStatus().isJobComplete()) {
    UtilsForTests.waitFor(100);
    jInfo = remoteJTClient.getJobInfo(jobId);
  }
  Assert.assertEquals("JobStatus", JobStatus.SUCCEEDED, 
     jInfo.getStatus().getRunState());
}
 

import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
/**
 * Run the job with two distributed cache files and verify
 * whether job is succeeded or not.
 * @throws Exception
 */
@Test
public void testCacheFilesLocalization() throws Exception {
  conf = wovenClient.getDaemonConf();
  SleepJob job = new SleepJob();
  job.setConf(conf);
  JobConf jobConf = job.setupJobConf(4, 1, 4000, 4000, 1000, 1000);
  DistributedCache.createSymlink(jobConf);
  DistributedCache.addCacheFile(cacheFileURI1, jobConf);
  DistributedCache.addCacheFile(cacheFileURI2, jobConf);
  RunningJob runJob = jobClient.submitJob(jobConf);
  JobID jobId = runJob.getID();

  Assert.assertTrue("Job has not been started for 1 min.", 
      jtClient.isJobStarted(jobId));
  TaskInfo[] taskInfos = wovenClient.getTaskInfo(jobId);
  Assert.assertTrue("Cache File1 has not been localize",
      checkLocalization(taskInfos,cacheFile1));
  Assert.assertTrue("Cache File2 has not been localize",
          checkLocalization(taskInfos,cacheFile2));
  JobInfo jInfo = wovenClient.getJobInfo(jobId);
  LOG.info("Waiting till the job is completed...");
  while (!jInfo.getStatus().isJobComplete()) {
    UtilsForTests.waitFor(100);
    jInfo = wovenClient.getJobInfo(jobId);
  }
  Assert.assertEquals("Job has not been succeeded", 
      jInfo.getStatus().getRunState(), JobStatus.SUCCEEDED);
}
 

import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
/**
 * Run the job with distributed cache files and remove one cache
 * file from the DFS when it is localized.verify whether the job
 * is failed or not.
 * @throws Exception
 */
@Test
public void testDeleteCacheFileInDFSAfterLocalized() throws Exception {
  conf = wovenClient.getDaemonConf();
  SleepJob job = new SleepJob();
  job.setConf(conf);
  JobConf jobConf = job.setupJobConf(4, 1, 4000, 4000, 1000, 1000);
  cacheFileURI3 = createCacheFile(tmpFolderPath, cacheFile3);
  DistributedCache.createSymlink(jobConf);
  DistributedCache.addCacheFile(cacheFileURI3, jobConf);
  RunningJob runJob = jobClient.submitJob(jobConf);
  JobID jobId = runJob.getID();
  Assert.assertTrue("Job has not been started for 1 min.", 
      jtClient.isJobStarted(jobId));
  TaskInfo[] taskInfos = wovenClient.getTaskInfo(jobId);
  boolean iscacheFileLocalized = checkLocalization(taskInfos,cacheFile3);
  Assert.assertTrue("CacheFile has not been localized", 
      iscacheFileLocalized);
  deleteCacheFile(new Path(tmpFolderPath, cacheFile3));
  JobInfo jInfo = wovenClient.getJobInfo(jobId);
  LOG.info("Waiting till the job is completed...");
  while (!jInfo.getStatus().isJobComplete()) {
    UtilsForTests.waitFor(100);
    jInfo = wovenClient.getJobInfo(jobId);
  }
  Assert.assertEquals("Job has not been failed", 
      jInfo.getStatus().getRunState(), JobStatus.FAILED);
}
 

import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
private RunningJob submitSleepJob(int numMappers, int numReducers, 
                                  long mapSleepTime, long reduceSleepTime,
                                  boolean shouldComplete, String userInfo,
                                  String queueName) 
                                    throws IOException {
  JobConf clientConf = new JobConf();
  clientConf.set("mapred.job.tracker", "localhost:"
      + miniMRCluster.getJobTrackerPort());
  SleepJob job = new SleepJob();
  job.setConf(clientConf);
  clientConf = job.setupJobConf(numMappers, numReducers, 
      mapSleepTime, (int)mapSleepTime/100,
      reduceSleepTime, (int)reduceSleepTime/100);
  if (queueName != null) {
    clientConf.setQueueName(queueName);
  }
  JobConf jc = new JobConf(clientConf);
  if (userInfo != null) {
    jc.set(UnixUserGroupInformation.UGI_PROPERTY_NAME, userInfo);
  }
  RunningJob rJob = null;
  if (shouldComplete) {
    rJob = JobClient.runJob(jc);  
  } else {
    rJob = new JobClient(clientConf).submitJob(jc);
  }
  return rJob;
}
 

import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
/**
 * Test case which checks if the jobs which fail initialization are removed
 * from the {@link CapacityTaskScheduler} waiting queue.
 * 
 * @throws Exception
 */
public void testFailingJobInitalization() throws Exception {
  Properties schedulerProps = new Properties();
  schedulerProps.put("mapred.capacity-scheduler.queue.default.capacity",
      "100");
  Properties clusterProps = new Properties();
  clusterProps.put("mapred.tasktracker.map.tasks.maximum", String.valueOf(1));
  clusterProps.put("mapred.tasktracker.reduce.tasks.maximum", String
      .valueOf(1));
  clusterProps.put("mapred.jobtracker.maxtasks.per.job", String.valueOf(1));
  // cluster capacity 1 maps, 1 reduces
  startCluster(1, clusterProps, schedulerProps);
  JobConf conf = getJobConf();
  conf.setSpeculativeExecution(false);
  conf.set("mapred.committer.job.setup.cleanup.needed", "false");
  conf.setNumTasksToExecutePerJvm(-1);
  SleepJob sleepJob = new SleepJob();
  sleepJob.setConf(conf);
  JobConf job = sleepJob.setupJobConf(3, 3, 1, 1, 1, 1);
  RunningJob rjob;
  try {
    rjob = runJob(job, false);
    fail("The job should have thrown Exception");
  } catch (Exception e) {
    CapacityTaskScheduler scheduler = (CapacityTaskScheduler) getJobTracker()
        .getTaskScheduler();
    JobQueuesManager mgr = scheduler.jobQueuesManager;
    assertEquals("Failed job present in Waiting queue", 0, mgr
        .getWaitingJobCount("default"));
  }
}
 

import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
@Test
public void testJobSubmission() throws Exception {
  Configuration conf = new Configuration(cluster.getConf());
  SleepJob job = new SleepJob();
  job.setConf(conf);
  conf = job.setupJobConf(1, 1, 100, 100, 100, 100);
  RunningJob rJob = cluster.getJTClient().submitAndVerifyJob(conf);
  cluster.getJTClient().verifyJobHistory(rJob.getID());
}
 

import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
/**
 * Test case which checks if the jobs which fail initialization are removed
 * from the {@link CapacityTaskScheduler} waiting queue.
 * 
 * @throws Exception
 */
@Test
public void testFailingJobInitalization() throws Exception {
  Properties schedulerProps = new Properties();
  schedulerProps.put("mapred.capacity-scheduler.queue.default.capacity",
      "100");
  Properties clusterProps = new Properties();
  clusterProps.put("mapred.tasktracker.map.tasks.maximum", String.valueOf(1));
  clusterProps.put("mapred.tasktracker.reduce.tasks.maximum", String
      .valueOf(1));
  clusterProps.put("mapred.jobtracker.maxtasks.per.job", String.valueOf(1));
  // cluster capacity 1 maps, 1 reduces
  startCluster(1, clusterProps, schedulerProps);
  JobConf conf = getJobConf();
  conf.setSpeculativeExecution(false);
  conf.set("mapred.committer.job.setup.cleanup.needed", "false");
  conf.setNumTasksToExecutePerJvm(-1);
  SleepJob sleepJob = new SleepJob();
  sleepJob.setConf(conf);
  JobConf job = sleepJob.setupJobConf(3, 3, 1, 1, 1, 1);
  RunningJob rjob;
  try {
    rjob = runJob(job, false);
    fail("The job should have thrown Exception");
  } catch (Exception e) {
    CapacityTaskScheduler scheduler = (CapacityTaskScheduler) getJobTracker()
        .getTaskScheduler();
    JobQueuesManager mgr = scheduler.jobQueuesManager;
    assertEquals("Failed job present in Waiting queue", 0, mgr
        .getQueue("default").getNumWaitingJobs());
  }
}
 

import org.apache.hadoop.examples.SleepJob; //导入方法依赖的package包/类
@Test
public void testJobSummaryInfoForDifferentUser() throws Exception {
  UserGroupInformation proxyUGI;
  UserGroupInformation ugi = UserGroupInformation.getLoginUser();
  ArrayList<String> users = cluster.getHadoopMultiUsersList();
  Assert.assertTrue("proxy users are not found.", users.size() > 0);
  if (conf.get("hadoop.security.authentication").equals("simple")) {
    proxyUGI = UserGroupInformation.createRemoteUser(
        users.get(0));
  } else {
    proxyUGI = UserGroupInformation.createProxyUser(
    users.get(0), ugi);
  }
  SleepJob job = new SleepJob();
  job.setConf(conf);
  final JobConf jobConf = job.setupJobConf(2, 1, 2000, 2000, 100, 100);
  final JobClient jClient =
  	proxyUGI.doAs(new PrivilegedExceptionAction<JobClient>() {
        public JobClient run() throws IOException {
          return new JobClient(jobConf);
        }
      });
  RunningJob runJob = proxyUGI.doAs(
      new PrivilegedExceptionAction<RunningJob>() {
    public RunningJob run() throws IOException {
      return jClient.submitJob(jobConf);
    }
  });
  JobID jobId = runJob.getID();
  Assert.assertTrue("Job has not been started for 1 min.", 
      jtClient.isJobStarted(jobId));
  LOG.info("Waiting till the job is completed...");
  Assert.assertTrue("Job has not been completed for 1 min.",
      jtClient.isJobStopped(jobId));
  JobInfo jInfo = remoteJTClient.getJobInfo(jobId);
  Assert.assertEquals("Job has not been succeeded", 
      jInfo.getStatus().getRunState(), JobStatus.SUCCEEDED);
   verifyJobSummaryInfo(jInfo,jobId);  
}