Java PigStats.get方法代码示例

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
private PigStats exec(String query) throws IOException {
    LOG.info("Query to run:\n" + query);
    List<PigProgressNotificationListener> listeners = ScriptState.get()
            .getAllListeners();
    ScriptState.start("embedded", scriptContext.getPigContext());
    ScriptState.get().setScript(query);
    for (PigProgressNotificationListener listener : listeners) {
        ScriptState.get().registerListener(listener);
    }
    PigServer pigServer = new PigServer(scriptContext.getPigContext(), false);
    pigServer.setBatchOn();
    GruntParser grunt = new GruntParser(new StringReader(query));
    grunt.setInteractive(false);
    grunt.setParams(pigServer);
    try {
        grunt.parseStopOnError(true);
    } catch (ParseException e) {
        throw new IOException("Failed to parse script " + e.getMessage(), e);
    }
    pigServer.executeBatch();
    return PigStats.get();
}
 

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
@Override
public PigStats call() throws Exception {
    LOG.info("Query to run:\n" + query);          
    ScriptState.start("embedded", scriptContext.getPigContext());
    ScriptState.get().setScript(query);
    ScriptState.get().registerListener(adaptor);
    PigServer pigServer = new PigServer(ctx, true);
    pigServer.setBatchOn();
    GruntParser grunt = new GruntParser(new StringReader(query));
    grunt.setInteractive(false);
    grunt.setParams(pigServer);
    try {
        grunt.parseStopOnError(true);
    } catch (ParseException e) {
        throw new IOException("Failed to parse script", e);
    }
    pigServer.executeBatch();
    return PigStats.get();
}
 

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
/**
 * Submits a batch of Pig commands for execution. Parse and build of script
 * should be skipped if user called {@link PigServer#parseAndBuild()}
 * before. Pass false as an argument in which case.
 *
 * @param parseAndBuild
 * @return
 * @throws IOException
 */
public List<ExecJob> executeBatch(boolean parseAndBuild) throws IOException {
    if (parseAndBuild) {
        parseAndBuild();
    }

    PigStats stats = null;

    if( !isMultiQuery ) {
        // ignore if multiquery is off
        stats = PigStats.get();
    } else {
        stats = execute();
    }

    return getJobs(stats);
}
 

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
/**
 * Compile and execute the current plan.
 * @return
 * @throws IOException
 */
private PigStats execute() throws IOException {
    pigContext.getProperties().setProperty( PigContext.JOB_NAME, jobName );
    if( jobPriority != null ) {
        pigContext.getProperties().setProperty( PigContext.JOB_PRIORITY, jobPriority );
    }

    // In this plan, all stores in the plan will be executed. They should be ignored if the plan is reused.
    currDAG.countExecutedStores();

    currDAG.compile();

    if( currDAG.lp.size() == 0 ) {
       return PigStats.get();
    }

    pigContext.getProperties().setProperty("pig.logical.plan.signature", currDAG.lp.getSignature());

    PigStats stats = executeCompiledLogicalPlan();

    return stats;
}
 

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
private PigStats exec(String query) throws IOException {
    LOG.info("Query to run:\n" + query);
    List<PigProgressNotificationListener> listeners = ScriptState.get().getAllListeners();
    PigContext pc = scriptContext.getPigContext();
    ScriptState scriptState = pc.getExecutionEngine().instantiateScriptState();
    ScriptState.start(scriptState);
    ScriptState.get().setScript(query);
    for (PigProgressNotificationListener listener : listeners) {
        ScriptState.get().registerListener(listener);
    }
    PigServer pigServer = new PigServer(scriptContext.getPigContext(), false);
    pigServer.setBatchOn();
    GruntParser grunt = new GruntParser(new StringReader(query), pigServer);
    grunt.setInteractive(false);
    try {
        grunt.parseStopOnError(true);
    } catch (ParseException e) {
        throw new IOException("Failed to parse script " + e.getMessage(), e);
    }
    pigServer.executeBatch();
    return PigStats.get();
}
 

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
@Override
public PigStats call() throws Exception {
    LOG.info("Query to run:\n" + query);
    PigContext pc = scriptContext.getPigContext();
    ScriptState scriptState = pc.getExecutionEngine().instantiateScriptState();
    ScriptState.start(scriptState);
    ScriptState.get().setScript(query);
    ScriptState.get().registerListener(adaptor);
    PigServer pigServer = new PigServer(ctx, true);
    pigServer.setBatchOn();
    GruntParser grunt = new GruntParser(new StringReader(query), pigServer);
    grunt.setInteractive(false);
    try {
        grunt.parseStopOnError(true);
    } catch (ParseException e) {
        throw new IOException("Failed to parse script", e);
    }
    pigServer.executeBatch();
    return PigStats.get();
}
 

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
/**
 *
 * @param location
 * @return the data in this location
 */
public List<Tuple> get(String location) {
  if (!locationToData.containsKey(location)) {
      PigStats pigStats = PigStats.get();
      boolean successful = pigStats.isSuccessful();
      throw new RuntimeException("No data for location '" + location + "'." + (successful ? "" : " It seems the job has failled, check logs"));
  }
  return locationToData.get(location).getAll();
}
 

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
private void executeBatch() throws IOException {
    if (mPigServer.isBatchOn()) {
        if (mExplain != null) {
            explainCurrentBatch();
        }

        if (!mLoadOnly) {
            mPigServer.executeBatch();
            PigStats stats = PigStats.get();
            JobGraph jg = stats.getJobGraph();
            Iterator<JobStats> iter = jg.iterator();
            while (iter.hasNext()) {
                JobStats js = iter.next();
                if (!js.isSuccessful()) {
                    mNumFailedJobs++;
                    Exception exp = (js.getException() != null) ? js.getException()
                            : new ExecException(
                                    "Job failed, hadoop does not return any error message",
                                    2244);
                    LogUtils.writeLog(exp,
                            mPigServer.getPigContext().getProperties().getProperty("pig.logfile"),
                            log,
                            "true".equalsIgnoreCase(mPigServer.getPigContext().getProperties().getProperty("verbose")),
                            "Pig Stack Trace");
                } else {
                    mNumSucceededJobs++;
                }
            }
        }
    }
}
 

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
/**
 * Starts collecting statistics for the given MR plan
 *
 * @param pc the Pig context
 * @param client the Hadoop job client
 * @param jcc the job compiler
 * @param plan the MR plan
 */
public static void startCollection(PigContext pc, JobClient client,
        JobControlCompiler jcc, MROperPlan plan) {
    SimplePigStats ps = (SimplePigStats)PigStats.get();
    ps.initialize(pc, client, jcc, plan);

    MRScriptState.get().emitInitialPlanNotification(plan);
    MRScriptState.get().emitLaunchStartedNotification(plan.size());
}
 

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
/**
 * Stops collecting statistics for a MR plan
 *
 * @param display if true, log collected statistics in the Pig log
 *      file at INFO level
 */
public static void stopCollection(boolean display) {
    SimplePigStats ps = (SimplePigStats)PigStats.get();
    ps.finish();
    if (!ps.isSuccessful()) {
        LOG.error(ps.getNumberFailedJobs() + " map reduce job(s) failed!");
        String errMsg = ps.getErrorMessage();
        if (errMsg != null) {
            LOG.error("Error message: " + errMsg);
        }
    }
    MRScriptState.get().emitLaunchCompletedNotification(
            ps.getNumberSuccessfulJobs());
    if (display) ps.display();
}
 

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
private void executeBatch() throws IOException {
    if (mPigServer.isBatchOn()) {
        if (mExplain != null) {
            explainCurrentBatch();
        }

        if (!mLoadOnly) {
            mPigServer.executeBatch();
            PigStats stats = PigStats.get();
            JobGraph jg = stats.getJobGraph();
            Iterator<JobStats> iter = jg.iterator();
            while (iter.hasNext()) {
                JobStats js = iter.next();
                if (!js.isSuccessful()) {
                    mNumFailedJobs++;
                    Exception exp = (js.getException() != null) ? js.getException()
                            : new ExecException(
                                    "Job " + (js.getJobId() == null ? "" : js.getJobId() + " ") +
                                    "failed, hadoop does not return any error message",
                                    2244);
                    LogUtils.writeLog(exp,
                            mPigServer.getPigContext().getProperties().getProperty("pig.logfile"),
                            log,
                            "true".equalsIgnoreCase(mPigServer.getPigContext().getProperties().getProperty("verbose")),
                            "Pig Stack Trace");
                } else {
                    mNumSucceededJobs++;
                }
            }
        }
    }
}
 

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
@Test
public void testRun_setsErrorThrowableOnPigStats() {
    File outputFile = null;
    try {
        String filename = this.getClass().getSimpleName() + "_" + "testRun_setsErrorThrowableOnPigStats";
        outputFile = File.createTempFile(filename, ".out");
        outputFile.delete();
        
        File scriptFile = File.createTempFile(filename, ".pig");
        BufferedWriter bw = new BufferedWriter(new FileWriter(scriptFile));
        bw.write("a = load 'test/org/apache/pig/test/data/passwd';\n");
        bw.write("b = group a by $0\n");
        bw.write("c = foreach b generate group, COUNT(a) as cnt;\n");
        bw.write("store c into 'out'\n");
        bw.close();
        
        Main.run(new String[]{"-x", "local", scriptFile.getAbsolutePath()}, null);
        PigStats stats = PigStats.get();
        
        Throwable t = stats.getErrorThrowable();
        assertTrue(t instanceof FrontendException);
        
        FrontendException fe = (FrontendException) t;
        SourceLocation sl = fe.getSourceLocation();
        assertEquals(2, sl.line());
        assertEquals(15, sl.offset());
        
        Throwable cause = fe.getCause();
        assertTrue(cause instanceof ParserException);
        
    } catch (Exception e) {
        log.error("Encountered exception", e);
        fail("Encountered Exception");
    }
}
 

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
@Test
public void testRun_setsErrorThrowableForParamSubstitution() {
    File outputFile = null;
    try {
        String filename = this.getClass().getSimpleName() + "_" + "testRun_setsErrorThrowableForParamSubstitution";
        outputFile = File.createTempFile(filename, ".out");
        outputFile.delete();

        File scriptFile = File.createTempFile(filename, ".pig");
        BufferedWriter bw = new BufferedWriter(new FileWriter(scriptFile));
        bw.write("a = load '$NOEXIST';\n");
        bw.write("b = group a by $0\n");
        bw.write("c = foreach b generate group, COUNT(a) as cnt;\n");
        bw.write("store c into 'out'\n");
        bw.close();
        
        Main.run(new String[]{"-x", "local", scriptFile.getAbsolutePath()}, null);
        PigStats stats = PigStats.get();
        
        Throwable t = stats.getErrorThrowable();
        assertTrue(t instanceof IOException);
        
        Throwable cause = t.getCause();
        assertTrue(cause instanceof ParameterSubstitutionException);

        ParameterSubstitutionException pse = (ParameterSubstitutionException) cause;
        assertTrue(pse.getMessage().contains("NOEXIST"));
    } catch (Exception e) {
        log.error("Encountered exception", e);
        fail("Encountered Exception");
    }
}
 

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
/**
 * Updates the {@link JobGraph} of the {@link PigStats}. The initial
 * {@link JobGraph} is created without job ids using {@link MROperPlan},
 * before any job is submitted for execution. The {@link JobGraph} then
 * is updated with job ids after jobs are executed.
 *
 * @param jobMroMap the map that maps {@link Job}s to {@link MapReduceOper}s
 */
public static void updateJobMroMap(Map<Job, MapReduceOper> jobMroMap) {
    SimplePigStats ps = (SimplePigStats)PigStats.get();
    for (Map.Entry<Job, MapReduceOper> entry : jobMroMap.entrySet()) {
        MapReduceOper mro = entry.getValue();
        ps.mapMROperToJob(mro, entry.getKey());
    }
}
 

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
/**
 * Add stats for a new Job, which doesn't yet need to be completed.
 *
 * @param job the job being run
 * @return JobStats for the job
 */
public static JobStats addJobStats(Job job) {
    SimplePigStats ps = (SimplePigStats)PigStats.get();
    return ps.addMRJobStats(job);
}