Java ControlledJob类代码示例

import org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob; //导入依赖的package包/类
public int run(String[] args) throws Exception {
    System.out.println(Arrays.toString(args));
   /* getting the chunk of data and converting to corresponding Key Value Pairs */
    @SuppressWarnings("deprecation")
    String intermediateFileDir = "tmp";
    String intermediateFileDirFile =intermediateFileDir +"/part-r-00000";
    JobControl control = new JobControl("ChainMapReduce");
    ControlledJob step1 = new ControlledJob(jobListFriends(args[0], intermediateFileDir), null);
    ControlledJob step2 = new ControlledJob(jobRecommendFriends(intermediateFileDirFile, args[1]), Arrays.asList(step1));
    control.addJob(step1);
    control.addJob(step2);
    Thread workFlowThread =  new Thread(control, "workflowthread");
    workFlowThread.setDaemon(true);
    workFlowThread.start();  
    return 0;
}
 

import org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob; //导入依赖的package包/类
public void addJob(int numReducers, boolean mapoutputCompressed,
    boolean outputCompressed, Size size, JobControl gridmix) {
  final String prop = String.format("combiner.%sJobs.inputFiles", size);
  final String indir = getInputDirsFor(prop, size.defaultPath(VARCOMPSEQ));
  final String outdir = addTSSuffix("perf-out/combiner-out-dir-" + size);

  StringBuffer sb = new StringBuffer();
  sb.append("-r ").append(numReducers).append(" ");
  sb.append("-indir ").append(indir).append(" ");
  sb.append("-outdir ").append(outdir);
  sb.append("-mapoutputCompressed ");
  sb.append(mapoutputCompressed).append(" ");
  sb.append("-outputCompressed ").append(outputCompressed);

  String[] args = sb.toString().split(" ");
  clearDir(outdir);
  try {
    Job job = CombinerJobCreator.createJob(args);
    job.setJobName("GridmixCombinerJob." + size);
    ControlledJob cjob = new ControlledJob(job, null);
    gridmix.addJob(cjob);
  } catch (Exception ex) {
    ex.printStackTrace();
  }
}
 

import org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob; //导入依赖的package包/类
static ArrayList<Job> castToJobList(List<ControlledJob> cjobs) {
  ArrayList<Job> ret = new ArrayList<Job>();
  for (ControlledJob job : cjobs) {
    ret.add((Job)job);
  }
  return ret;
}
 

import org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob; //导入依赖的package包/类
public static JobControl createValueAggregatorJobs(String args[],
  Class<? extends ValueAggregatorDescriptor>[] descriptors) 
throws IOException {
  
  JobControl theControl = new JobControl("ValueAggregatorJobs");
  ArrayList<ControlledJob> dependingJobs = new ArrayList<ControlledJob>();
  Configuration conf = new Configuration();
  if (descriptors != null) {
    conf = setAggregatorDescriptors(descriptors);
  }
  Job job = createValueAggregatorJob(conf, args);
  ControlledJob cjob = new ControlledJob(job, dependingJobs);
  theControl.addJob(cjob);
  return theControl;
}
 

import org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob; //导入依赖的package包/类
/**
 * Add new job to JobControl instance.
 */
public synchronized void addJob(String[] args) throws IOException {
    Job job = createJob(args);
    this.jc.addJob(new ControlledJob(job, null));
    if(this.jobIndexMap.containsKey(job.getJobName())) {
        throw new IllegalStateException("Job name should be unique. please check name with: " + job.getJobName());
    }
    this.jobIndexMap.put(job.getJobName(), this.jobIndex);
    this.jobIndexParams.put(this.jobIndex, args);
    this.jobRunningTimes.put(this.jobIndex, 1);
    this.jobIndex += 1;
}
 

import org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob; //导入依赖的package包/类
public String toFakedStateString(ControlledJob controlledJob) {
    StringBuffer sb = new StringBuffer();
    sb.append("job name:\t").append(controlledJob.getJob().getJobName()).append("\n");
    sb.append("job id:\t").append(controlledJob.getJobID()).append("\n");
    sb.append("job state:\t").append("SUCCESS").append("\n");
    sb.append("job mapred id:\t").append(controlledJob.getJob().getJobID()).append("\n");
    sb.append("job message:\t").append(" successful job").append("\n");
    sb.append("job has no depending job:\t").append("\n");
    return sb.toString();
}
 

import org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob; //导入依赖的package包/类
private State checkState(ControlledJob j) {
  try {
    return (State)checkState.invoke(j);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

import org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob; //导入依赖的package包/类
private State submit(ControlledJob j) {
  try {
    return (State)submit.invoke(j);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

import org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob; //导入依赖的package包/类
@SuppressWarnings("unchecked")
private LinkedList<ControlledJob> getJobs(Field field) {
  try {
    return (LinkedList<ControlledJob>)field.get(this);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

import org.apache.hadoop.mapreduce.lib.jobcontrol.ControlledJob; //导入依赖的package包/类
/** 
 * Construct a job.
 * @param jobConf a mapred job configuration representing a job to be executed.
 * @param dependingJobs an array of jobs the current job depends on
 */
@SuppressWarnings("unchecked")
public Job(JobConf jobConf, ArrayList<?> dependingJobs) throws IOException {
  super(org.apache.hadoop.mapreduce.Job.getInstance(jobConf),
        (List<ControlledJob>) dependingJobs);
}