Java HExecutionEngine类代码示例

import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine; //导入依赖的package包/类
public void setParams(PigServer pigServer)
{
    mPigServer = pigServer;

    mDfs = mPigServer.getPigContext().getDfs();
    mLfs = mPigServer.getPigContext().getLfs();
    mConf = mPigServer.getPigContext().getProperties();
    shell = new FsShell(ConfigurationUtil.toConfiguration(mConf));

    // TODO: this violates the abstraction layer decoupling between
    // front end and back end and needs to be changed.
    // Right now I am not clear on how the Job Id comes from to tell
    // the back end to kill a given job (mJobClient is used only in
    // processKill)
    //
    HExecutionEngine execEngine = mPigServer.getPigContext().getExecutionEngine();
    mJobConf = execEngine.getJobConf();
}
 

import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine; //导入依赖的package包/类
public ExampleGenerator(LogicalPlan plan, PigContext hadoopPigContext) {
        this.plan = plan;
//        pigContext = new PigContext(ExecType.LOCAL, hadoopPigContext
//                .getProperties());
        pigContext = hadoopPigContext;
        // pigContext.setExecType(ExecType.LOCAL);
        FileLocalizer.setInitialized(false);
        try {
            pigContext.connect();
        } catch (ExecException e) {
            log.error("Error connecting to the cluster "
                    + e.getLocalizedMessage());

        }
        execEngine = new HExecutionEngine(pigContext);
        localMRRunner = new LocalMapReduceSimulator();
        poLoadToSchemaMap = new HashMap<POLoad, LogicalSchema>();
    }
 

import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine; //导入依赖的package包/类
public void setParams(PigServer pigServer)
{
    mPigServer = pigServer;
    
    mDfs = mPigServer.getPigContext().getDfs();
    mLfs = mPigServer.getPigContext().getLfs();
    mConf = mPigServer.getPigContext().getProperties();
    shell = new FsShell(ConfigurationUtil.toConfiguration(mConf));
    
    // TODO: this violates the abstraction layer decoupling between
    // front end and back end and needs to be changed.
    // Right now I am not clear on how the Job Id comes from to tell
    // the back end to kill a given job (mJobClient is used only in 
    // processKill)
    //
    HExecutionEngine execEngine = mPigServer.getPigContext().getExecutionEngine();
    mJobConf = execEngine.getJobConf();
}
 

import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine; //导入依赖的package包/类
public void connect() throws ExecException {

        switch (execType) {
            case LOCAL:
            case MAPREDUCE:
            {
                executionEngine = new HExecutionEngine (this);

                executionEngine.init();
                
                dfs = executionEngine.getDataStorage();
                
                lfs = new HDataStorage(URI.create("file:///"),
                                        properties); 
            }
            break;
            
            default:
            {
                int errCode = 2040;
                String msg = "Unkown exec type: " + execType;
                throw new ExecException(msg, errCode, PigException.BUG);
            }
        }

    }
 

import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine; //导入依赖的package包/类
public ExampleGenerator(LogicalPlan plan, PigContext hadoopPigContext) {
        this.plan = plan;
//        pigContext = new PigContext(ExecType.LOCAL, hadoopPigContext
//                .getProperties());
        pigContext = hadoopPigContext;
        //Kaituo
 //       pigContext.setDefaultLogLevel(Level.FATAL);
        // pigContext.setExecType(ExecType.LOCAL);
        FileLocalizer.setInitialized(false);
        try {
            pigContext.connect();
        } catch (ExecException e) {
            log.error("Error connecting to the cluster "
                    + e.getLocalizedMessage());

        }
        execEngine = new HExecutionEngine(pigContext);
        localMRRunner = new LocalMapReduceSimulator();
        poLoadToSchemaMap = new HashMap<POLoad, LogicalSchema>();
    }
 

import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine; //导入依赖的package包/类
@Override
public void cancel(InterpreterContext context) {
  LOGGER.info("Cancel paragraph:" + context.getParagraphId());
  PigScriptListener listener = listenerMap.get(context.getParagraphId());
  if (listener != null) {
    Set<String> jobIds = listener.getJobIds();
    if (jobIds.isEmpty()) {
      LOGGER.info("No job is started, so can not cancel paragraph:" + context.getParagraphId());
    }
    for (String jobId : jobIds) {
      LOGGER.info("Kill jobId:" + jobId);
      HExecutionEngine engine =
              (HExecutionEngine) getPigServer().getPigContext().getExecutionEngine();
      try {
        Field launcherField = HExecutionEngine.class.getDeclaredField("launcher");
        launcherField.setAccessible(true);
        Launcher launcher = (Launcher) launcherField.get(engine);
        // It doesn't work for Tez Engine due to PIG-5035
        launcher.killJob(jobId, new Configuration());
      } catch (NoSuchFieldException | BackendException | IllegalAccessException e) {
        LOGGER.error("Fail to cancel paragraph:" + context.getParagraphId(), e);
      }
    }
  } else {
    LOGGER.warn("No PigScriptListener found, can not cancel paragraph:"
            + context.getParagraphId());
  }
}
 

import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine; //导入依赖的package包/类
/**
 * Given a (modified) new logical plan, run the script.
 * @param newPlan plan to run
 * @param jobName name to give the MR jobs associated with this run
 * @return list of exec jobs describing the jobs that were run.
 * @throws FrontendException if plan translation fails.
 * @throws ExecException if running the job fails.
 */
public List<ExecJob> runPlan(LogicalPlan newPlan,
                             String jobName) throws FrontendException, ExecException {
	
    HExecutionEngine engine = new HExecutionEngine(pigContext);
    PhysicalPlan pp = engine.compile(newPlan, null);
    PigStats stats = launchPlan(pp, jobName);
    return getJobs(stats);                        
}
 

import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine; //导入依赖的package包/类
public void connect() throws ExecException {

        switch (execType) {
            case LOCAL:
            case SPARK:
            case SPARKSTREAMING:
            case MAPREDUCE:
            {
                executionEngine = new HExecutionEngine (this);

                executionEngine.init();

                dfs = executionEngine.getDataStorage();

                lfs = new HDataStorage(URI.create("file:///"),
                                        properties);
            }
            break;

            default:
            {
                int errCode = 2040;
                String msg = "Unkown exec type: " + execType;
                throw new ExecException(msg, errCode, PigException.BUG);
            }
        }

    }
 

import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine; //导入依赖的package包/类
public static PhysicalPlan buildPp(PigServer pigServer, String query)
throws Exception {
    LogicalPlan lp = buildLp( pigServer, query );
    lp.optimize(pigServer.getPigContext());
    return ((HExecutionEngine)pigServer.getPigContext().getExecutionEngine()).compile(lp,
            pigServer.getPigContext().getProperties());
}
 

import org.apache.pig.backend.hadoop.executionengine.HExecutionEngine; //导入依赖的package包/类
public HExecutionEngine getExecutionEngine() {
    return executionEngine;
}