Java JobID類代碼示例

import org.apache.hadoop.mapred.JobID; //導入依賴的package包/類
@SuppressWarnings("deprecation")
@Test(timeout = 30000)
public void testJobState() throws Exception {
  Job job_1 = getCopyJob();
  JobControl jc = new JobControl("Test");
  jc.addJob(job_1);
  Assert.assertEquals(Job.WAITING, job_1.getState());
  job_1.setState(Job.SUCCESS);
  Assert.assertEquals(Job.WAITING, job_1.getState());

  org.apache.hadoop.mapreduce.Job mockjob =
      mock(org.apache.hadoop.mapreduce.Job.class);
  org.apache.hadoop.mapreduce.JobID jid =
      new org.apache.hadoop.mapreduce.JobID("test", 0);
  when(mockjob.getJobID()).thenReturn(jid);
  job_1.setJob(mockjob);
  Assert.assertEquals("job_test_0000", job_1.getMapredJobID());
  job_1.setMapredJobID("job_test_0001");
  Assert.assertEquals("job_test_0000", job_1.getMapredJobID());
  jc.stop();
}
 

import org.apache.hadoop.mapred.JobID; //導入依賴的package包/類
/**
 * Submit/run a map/reduce job.
 * 
 * @param job
 * @return true for success
 * @throws IOException
 */
public static boolean runJob(JobConf job) throws IOException {
  JobClient jc = new JobClient(job);
  boolean sucess = true;
  RunningJob running = null;
  try {
    running = jc.submitJob(job);
    JobID jobId = running.getID();
    System.out.println("Job " + jobId + " is submitted");
    while (!running.isComplete()) {
      System.out.println("Job " + jobId + " is still running.");
      try {
        Thread.sleep(60000);
      } catch (InterruptedException e) {
      }
      running = jc.getJob(jobId);
    }
    sucess = running.isSuccessful();
  } finally {
    if (!sucess && (running != null)) {
      running.killJob();
    }
    jc.close();
  }
  return sucess;
}
 

import org.apache.hadoop.mapred.JobID; //導入依賴的package包/類
public void testRemoveTaskDistributedCacheManager() throws Exception {
  if (!canRun()) {
    return;
  }
  TrackerDistributedCacheManager manager = new TrackerDistributedCacheManager(
      conf, taskController);
  JobID jobId = new JobID("jobtracker", 1);
  manager.newTaskDistributedCacheManager(jobId, conf);

  TaskDistributedCacheManager taskDistributedCacheManager = manager
      .getTaskDistributedCacheManager(jobId);
  assertNotNull(taskDistributedCacheManager);

  manager.removeTaskDistributedCacheManager(jobId);

  taskDistributedCacheManager = manager.getTaskDistributedCacheManager(jobId);
  assertNull(taskDistributedCacheManager);
}
 

import org.apache.hadoop.mapred.JobID; //導入依賴的package包/類
/**
  * Allow the job to continue through MR control job.
  * @param id of the job. 
  * @throws IOException when failed to get task info. 
  */
public void signalAllTasks(JobID id) throws IOException{
  TaskInfo[] taskInfos = getJTClient().getProxy().getTaskInfo(id);
  if(taskInfos !=null) {
    for (TaskInfo taskInfoRemaining : taskInfos) {
      if(taskInfoRemaining != null) {
        FinishTaskControlAction action = new FinishTaskControlAction(TaskID
            .downgrade(taskInfoRemaining.getTaskID()));
        Collection<TTClient> tts = getTTClients();
        for (TTClient cli : tts) {
          cli.getProxy().sendAction(action);
        }
      }
    }  
  }
}
 

import org.apache.hadoop.mapred.JobID; //導入依賴的package包/類
private Job submitAndValidateJob(JobConf conf, int numMaps, int numReds)
    throws IOException, InterruptedException, ClassNotFoundException {
  conf.setJobSetupCleanupNeeded(false);
  Job job = MapReduceTestUtil.createJob(conf, inDir, outDir,
              numMaps, numReds);

  job.setOutputFormatClass(MyOutputFormat.class);
  job.waitForCompletion(true);
  assertTrue(job.isSuccessful());
  JobID jobid = (org.apache.hadoop.mapred.JobID)job.getID();

  JobClient jc = new JobClient(conf);
  assertTrue(jc.getSetupTaskReports(jobid).length == 0);
  assertTrue(jc.getCleanupTaskReports(jobid).length == 0);
  assertTrue(jc.getMapTaskReports(jobid).length == numMaps);
  assertTrue(jc.getReduceTaskReports(jobid).length == numReds);
  FileSystem fs = FileSystem.get(conf);
  assertTrue("Job output directory doesn't exit!", fs.exists(outDir));
  FileStatus[] list = fs.listStatus(outDir, new OutputFilter());
  int numPartFiles = numReds == 0 ? numMaps : numReds;
  assertTrue("Number of part-files is " + list.length + " and not "
      + numPartFiles, list.length == numPartFiles);
  return job;
}
 

import org.apache.hadoop.mapred.JobID; //導入依賴的package包/類
public void testNewTag() throws IOException {
  LOG.info("Start testNewTag");
  JobID jobid = new JobID("TestJob", 1);
  long oldTimeStamp = releaseTimeStamp;
  long currentTimeStamp = System.currentTimeMillis();
  try {
    Thread.sleep(1000);
  } catch(InterruptedException e) {
  }
  String workingPath = getRelease(releaseTimeStamp, jobid);
  String workingTag = workingPath + "/RELEASE_COPY_DONE";
  FileStatus tagStatus = fs.getFileStatus(new Path(workingTag));
  long newTimeStamp = tagStatus.getModificationTime();
  LOG.info("Before getRelease, " + workingTag + " timestamp is " + oldTimeStamp);
  LOG.info("After getRelease, the timestamp is " + newTimeStamp);
  assertEquals(newTimeStamp > currentTimeStamp, true);
  assertEquals(newTimeStamp > oldTimeStamp, true);
  LOG.info("Done with the testing for testNewTag");
}
 

import org.apache.hadoop.mapred.JobID; //導入依賴的package包/類
@Override
public RunningJob submitJob(final JobConf job) throws IOException
{
    ensureInvocationGridPresent();
    ExecutorService async = Executors.newSingleThreadExecutor();
    final JobID jobID = JobID.forName("job_"+job.getJobName()+"_0");

    Future jobSubmitted = async.submit(new Callable<Object>() {
        @Override
        public Object call() throws Exception {
            try {
                JobScheduler.getInstance().runOldApiJob(job, jobID, sortEnabled, null, grid);
            } finally {
                if (unloadGrid) {
                    grid.unload();
                }
            }
            return null;
        }
    });
    async.shutdown(); //Will shut down after task is done

    return new HServerRunningJob(jobID, jobSubmitted);
}
 

import org.apache.hadoop.mapred.JobID; //導入依賴的package包/類
private void cleanRecovery() throws IOException {
  new LightWeightRequestHandler(HDFSOperationType.DELETE_ENCODING_JOBS) {
    @Override
    public Object performTask() throws IOException {
      EncodingJobsDataAccess da = (EncodingJobsDataAccess)
          HdfsStorageFactory.getDataAccess(EncodingJobsDataAccess.class);
      Iterator<MapReduceEncoder> it = completedJobs.iterator();
      while (it.hasNext()) {
        MapReduceEncoder job = it.next();
        JobID jobId = job.getJobID();
        da.delete(new EncodingJob(jobId.getJtIdentifier(), jobId.getId()));
        it.remove();
      }
      return null;
    }
  }.handle();
}
 

import org.apache.hadoop.mapred.JobID; //導入依賴的package包/類
@Override
public void abortJob(JobContext context, JobStatus.State runState) throws java.io.IOException {
	super.abortJob(context, runState);

	final JobClient jobClient = new JobClient(new JobConf(context.getConfiguration()));
	final RunningJob job = jobClient.getJob((org.apache.hadoop.mapred.JobID) JobID.forName(context.getConfiguration().get("mapred.job.id")));
	String diag = "";
	for (final TaskCompletionEvent event : job.getTaskCompletionEvents(0))
		switch (event.getTaskStatus()) {
			case SUCCEEDED:
				break;
               default:
				diag += "Diagnostics for: " + event.getTaskTrackerHttp() + "\n";
				for (final String s : job.getTaskDiagnostics(event.getTaskAttemptId()))
					diag += s + "\n";
				diag += "\n";
				break;
		}
	updateStatus(diag, context.getConfiguration().getInt("boa.hadoop.jobid", 0));
}
 

import org.apache.hadoop.mapred.JobID; //導入依賴的package包/類
@SuppressWarnings("deprecation")
JobStats addJobStats(Job job) {
    MapReduceOper mro = jobMroMap.get(job);
     
    if (mro == null) {
        LOG.warn("unable to get MR oper for job: " + job.toString());
        return null;
    }
    JobStats js = mroJobMap.get(mro);
    
    JobID jobId = job.getAssignedJobID();
    js.setId(jobId);
    js.setAlias(mro);
    js.setConf(job.getJobConf());
    return js;
}
 

import org.apache.hadoop.mapred.JobID; //導入依賴的package包/類
@Override
protected void processKill(String jobid) throws IOException
{
    if (mJobConf != null) {
        JobClient jc = new JobClient(mJobConf);
        JobID id = JobID.forName(jobid);
        RunningJob job = jc.getJob(id);
        if (job == null)
            System.out.println("Job with id " + jobid + " is not active");
        else
        {
            job.killJob();
            log.info("Kill " + id + " submitted.");
        }
    }
}
 

import org.apache.hadoop.mapred.JobID; //導入依賴的package包/類
@Test
public void testMedianMapReduceTime() throws Exception {

	JobConf jobConf = new JobConf();
	JobClient jobClient = Mockito.mock(JobClient.class);
	
	// mock methods to return the predefined map and reduce task reports
	Mockito.when(jobClient.getMapTaskReports(jobID)).thenReturn(mapTaskReports);
	Mockito.when(jobClient.getReduceTaskReports(jobID)).thenReturn(reduceTaskReports);

	PigStats.JobGraph jobGraph = new PigStats.JobGraph();
	JobStats jobStats = createJobStats("JobStatsTest", jobGraph);
	getJobStatsMethod("setId", JobID.class).invoke(jobStats, jobID);
	getJobStatsMethod("setSuccessful", boolean.class).invoke(jobStats, true);

	getJobStatsMethod("addMapReduceStatistics", JobClient.class, Configuration.class)
	    .invoke(jobStats, jobClient, jobConf);
	String msg = (String)getJobStatsMethod("getDisplayString", boolean.class)
	    .invoke(jobStats, false);
	
	System.out.println(JobStats.SUCCESS_HEADER);
	System.out.println(msg);
	
	assertTrue(msg.startsWith(ASSERT_STRING));
}
 

import org.apache.hadoop.mapred.JobID; //導入依賴的package包/類
@Override
public void killJob(String jobID, Configuration conf) throws BackendException {
    try {
        if (conf != null) {
            JobConf jobConf = new JobConf(conf);
            JobClient jc = new JobClient(jobConf);
            JobID id = JobID.forName(jobID);
            RunningJob job = jc.getJob(id);
            if (job == null)
                System.out.println("Job with id " + jobID + " is not active");
            else
            {
                job.killJob();
                log.info("Kill " + id + " submitted.");
            }
        }
    } catch (IOException e) {
        throw new BackendException(e);
    }
}
 

import org.apache.hadoop.mapred.JobID; //導入依賴的package包/類
@Test
public void testMedianMapReduceTime() throws Exception {
    JobClient jobClient = Mockito.mock(JobClient.class);

    // mock methods to return the predefined map and reduce task reports
    Mockito.when(jobClient.getMapTaskReports(jobID)).thenReturn(mapTaskReports);
    Mockito.when(jobClient.getReduceTaskReports(jobID)).thenReturn(reduceTaskReports);

    PigStats.JobGraph jobGraph = new PigStats.JobGraph();
    MRJobStats jobStats = createJobStats("JobStatsTest", jobGraph);
    getJobStatsMethod("setId", JobID.class).invoke(jobStats, jobID);
    jobStats.setSuccessful(true);

    getJobStatsMethod("addMapReduceStatistics", Iterator.class, Iterator.class)
        .invoke(jobStats, Arrays.asList(mapTaskReports).iterator(), Arrays.asList(reduceTaskReports).iterator());
    String msg = (String)getJobStatsMethod("getDisplayString")
        .invoke(jobStats);

    System.out.println(JobStats.SUCCESS_HEADER);
    System.out.println(msg);

    assertTrue(msg.startsWith(ASSERT_STRING));
}
 

import org.apache.hadoop.mapred.JobID; //導入依賴的package包/類
@SuppressWarnings("deprecation")
@Test
public void testBadUpdate() throws Exception {
  JobStatus mockStatus = mock(JobStatus.class);
  JobProfile mockProf = mock(JobProfile.class);
  JobSubmissionProtocol mockClient = mock(JobSubmissionProtocol.class);
  
  JobID id = new JobID("test",0);
  
  RunningJob rj = new JobClient.NetworkedJob(mockStatus, mockProf, mockClient);
  
  when(mockProf.getJobID()).thenReturn(id);
  when(mockClient.getJobStatus(id)).thenReturn(null);
  
  boolean caught = false;
  try {
    rj.isSuccessful();
  } catch(IOException e) {
    caught = true;
  }
  assertTrue("Expected updateStatus to throw an IOException bt it did not", caught);
  
  //verification
  verify(mockProf).getJobID();
  verify(mockClient).getJobStatus(id);
}