Java OutputStats类代码示例

本文整理汇总了Java中org.apache.pig.tools.pigstats.OutputStats类的典型用法代码示例。如果您正苦于以下问题：Java OutputStats类的具体用法？Java OutputStats怎么用？Java OutputStats使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。

OutputStats类属于org.apache.pig.tools.pigstats包，在下文中一共展示了OutputStats类的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: getJobs

import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
/**
 * Retrieves a list of Job objects from the PigStats object
 * @param stats
 * @return A list of ExecJob objects
 */
protected List<ExecJob> getJobs(PigStats stats) {
    LinkedList<ExecJob> jobs = new LinkedList<ExecJob>();
    JobGraph jGraph = stats.getJobGraph();
    Iterator<JobStats> iter = jGraph.iterator();
    while (iter.hasNext()) {
        JobStats js = iter.next();
        for (OutputStats output : js.getOutputs()) {
            if (js.isSuccessful()) {
                jobs.add(new HJob(HJob.JOB_STATUS.COMPLETED, pigContext, output
                        .getPOStore(), output.getAlias(), stats));
            } else {
                HJob hjob = new HJob(HJob.JOB_STATUS.FAILED, pigContext, output
                        .getPOStore(), output.getAlias(), stats);
                hjob.setException(js.getException());
                jobs.add(hjob);
            }
        }
    }
    return jobs;
}

开发者ID:sigmoidanalytics，项目名称:spork-streaming，代码行数:26，代码来源:PigServer.java

示例2: addOneOutputStats

import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
private void addOneOutputStats(POStore sto) {
    long records = -1;
    if (sto.isMultiStore()) {
        Long n = multiStoreCounters.get(MRPigStatsUtil.getMultiStoreCounterName(sto));
        if (n != null) records = n;
    } else {
        records = mapOutputRecords;
    }

    long bytes = getOutputSize(sto, conf);
    String location = sto.getSFile().getFileName();
    OutputStats ds = new OutputStats(location, bytes, records,
            (state == JobState.SUCCESS));
    ds.setPOStore(sto);
    ds.setConf(conf);
    outputs.add(ds);

    if (state == JobState.SUCCESS) {
         MRScriptState.get().emitOutputCompletedNotification(ds);
    }
}

开发者ID:sigmoidanalytics，项目名称:spork，代码行数:22，代码来源:MRJobStats.java

示例3: accumulateStats

import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
/**
 * Updates the statistics after a DAG is finished.
 */
public void accumulateStats(TezJob tezJob) throws IOException {
    DAGStatus dagStatus = tezJob.getDAGStatus();
    TezDAGStats tezDAGStats = tezDAGStatsMap.get(tezJob.getName());
    if (dagStatus == null) {
        tezDAGStats.setSuccessful(false);
        tezScriptState.emitJobFailedNotification(tezDAGStats);
        return;
    } else {
        tezDAGStats.accumulateStats(tezJob);
        for(OutputStats output: tezDAGStats.getOutputs()) {
            tezScriptState.emitOutputCompletedNotification(output);
        }
        if (dagStatus.getState() == DAGStatus.State.SUCCEEDED) {
            tezDAGStats.setSuccessful(true);
            tezScriptState.emitjobFinishedNotification(tezDAGStats);
        } else if (dagStatus.getState() == DAGStatus.State.FAILED) {
            tezDAGStats.setSuccessful(false);
            tezDAGStats.setErrorMsg(tezJob.getDiagnostics());
            tezScriptState.emitJobFailedNotification(tezDAGStats);
        }
        tezScriptState.dagCompletedNotification(tezJob.getName(), tezDAGStats);
    }
}

开发者ID:sigmoidanalytics，项目名称:spork，代码行数:27，代码来源:TezPigScriptStats.java

示例4: outputCompletedNotification

import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Override
public void outputCompletedNotification(String scriptId,
        OutputStats outputStats) {
    synchronized (listeners) {
        for (PigProgressNotificationListener listener : listeners) {
            listener.outputCompletedNotification(scriptId, outputStats);
        }
    }        
}

开发者ID:sigmoidanalytics，项目名称:spork-streaming，代码行数:10，代码来源:SyncProgressNotificationAdaptor.java

示例5: testDuplicateCounterName2

import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Test
public void testDuplicateCounterName2() throws Exception {
    
    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = filter A by a0 > 3;");
    w1.println("store A into 'output';");
    w1.println("store B into 'tmp/output';");
    w1.close();
    
    try {
        String[] args = { PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener());
 
        assertTrue(stats.isSuccessful());
        
        assertEquals(1, stats.getNumberJobs());
        List<OutputStats> outputs = stats.getOutputStats();
        assertEquals(2, outputs.size());
        for (OutputStats outstats : outputs) {
            if (outstats.getLocation().endsWith("tmp/output")) {
                assertEquals(2, outstats.getNumberRecords());
            } else {
                assertEquals(5, outstats.getNumberRecords());
            }
        }
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
        Util.deleteFile(cluster, "tmp/output");
    }
}

开发者ID:sigmoidanalytics，项目名称:spork-streaming，代码行数:33，代码来源:TestPigRunner.java

示例6: testEmptyFileCounter2

import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Test //PIG-1893
public void testEmptyFileCounter2() throws Exception {

    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = filter A by a0 < 0;");
    w1.println("store A into '" + OUTPUT_FILE + "';");
    w1.println("store B into 'output2';");
    w1.close();
    
    try {
        String[] args = { PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener());
 
        assertTrue(stats.isSuccessful());
        
        assertEquals(1, stats.getNumberJobs());
        List<OutputStats> outputs = stats.getOutputStats();
        assertEquals(2, outputs.size());
        for (OutputStats outstats : outputs) {
            if (outstats.getLocation().endsWith("output2")) {
                assertEquals(0, outstats.getNumberRecords());
            } else {
                assertEquals(5, outstats.getNumberRecords());
            }
        }
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
        Util.deleteFile(cluster, "output2");
    }
}

开发者ID:sigmoidanalytics，项目名称:spork-streaming，代码行数:33，代码来源:TestPigRunner.java

示例7: testDisablePigCounters

import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Test // PIG-2208: Restrict number of PIG generated Haddop counters
public void testDisablePigCounters() throws Exception {        
    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("C = join A by a0, B by a0;");
    w1.println("store C into '" + OUTPUT_FILE + "';");
    w1.close();
    
    try {
        String[] args = { "-Dpig.disable.counter=true", PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener());
 
        assertTrue(stats.isSuccessful());
        
        assertEquals(1, stats.getNumberJobs());
        List<InputStats> inputs = stats.getInputStats();
        assertEquals(2, inputs.size());
        for (InputStats instats : inputs) {
            // the multi-input counters are disabled
            assertEquals(-1, instats.getNumberRecords());
        }
        
        List<OutputStats> outputs = stats.getOutputStats();
        assertEquals(1, outputs.size());
        OutputStats outstats = outputs.get(0);
        assertEquals(9, outstats.getNumberRecords());
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
    }
}

开发者ID:sigmoidanalytics，项目名称:spork-streaming，代码行数:33，代码来源:TestPigRunner.java

示例8: testDisablePigCounters2

import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Test // PIG-2208: Restrict number of PIG generated Haddop counters
public void testDisablePigCounters2() throws Exception {
    
    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = filter A by a0 > 3;");
    w1.println("store A into 'output';");
    w1.println("store B into 'tmp/output';");
    w1.close();
    
    try {
        String[] args = { "-Dpig.disable.counter=true", PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener());
 
        assertTrue(stats.isSuccessful());
        
        assertEquals(1, stats.getNumberJobs());
        List<OutputStats> outputs = stats.getOutputStats();
        assertEquals(2, outputs.size());
        for (OutputStats outstats : outputs) {
            // the multi-output counters are disabled
            assertEquals(-1, outstats.getNumberRecords());
        }
        
        List<InputStats> inputs = stats.getInputStats();
        assertEquals(1, inputs.size());
        InputStats instats = inputs.get(0);
        assertEquals(5, instats.getNumberRecords());
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
        Util.deleteFile(cluster, "tmp/output");
    }
}

开发者ID:sigmoidanalytics，项目名称:spork-streaming，代码行数:35，代码来源:TestPigRunner.java

示例9: outputCompletedNotification

import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Override
public void outputCompletedNotification(String scriptId,
        OutputStats outputStats) {
    synchronized (listeners) {
        for (PigProgressNotificationListener listener : listeners) {
            listener.outputCompletedNotification(scriptId, outputStats);
        }
    }
}

开发者ID:sigmoidanalytics，项目名称:spork，代码行数:10，代码来源:SyncProgressNotificationAdaptor.java

示例10: getDisplayString

import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Override
public String getDisplayString() {
    StringBuilder sb = new StringBuilder();
    String id = (jobId == null) ? "N/A" : jobId.toString();
    if (state == JobState.FAILED) {
        sb.append(id).append("\t")
            .append(getAlias()).append("\t")
            .append(getFeature()).append("\t");
        if (state == JobState.FAILED) {
            sb.append("Message: ").append(getErrorMessage()).append("\t");
        }
    } else if (state == JobState.SUCCESS) {
        sb.append(id).append("\t")
            .append(numberMaps).append("\t")
            .append(numberReduces).append("\t");
        appendStat(maxMapTime, sb);
        appendStat(minMapTime, sb);
        appendStat(avgMapTime, sb);
        appendStat(medianMapTime, sb);
        appendStat(maxReduceTime, sb);
        appendStat(minReduceTime, sb);
        appendStat(avgReduceTime, sb);
        appendStat(medianReduceTime, sb);

        sb.append(getAlias()).append("\t")
            .append(getFeature()).append("\t");
    }
    for (OutputStats os : outputs) {
        sb.append(os.getLocation()).append(",");
    }
    sb.append("\n");
    return sb.toString();
}

开发者ID:sigmoidanalytics，项目名称:spork，代码行数:34，代码来源:MRJobStats.java

示例11: getJobs

import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
/**
 * Retrieves a list of Job objects from the PigStats object
 * @param stats
 * @return A list of ExecJob objects
 */
protected List<ExecJob> getJobs(PigStats stats) {
    LinkedList<ExecJob> jobs = new LinkedList<ExecJob>();
    if (stats instanceof EmptyPigStats) {
        HJob job = new HJob(HJob.JOB_STATUS.COMPLETED, pigContext, stats.result(null)
                .getPOStore(), null);
        jobs.add(job);
        return jobs;
    }
    JobGraph jGraph = stats.getJobGraph();
    Iterator<JobStats> iter = jGraph.iterator();
    while (iter.hasNext()) {
        JobStats js = iter.next();
        for (OutputStats output : js.getOutputs()) {
            if (js.isSuccessful()) {
                jobs.add(new HJob(HJob.JOB_STATUS.COMPLETED, pigContext, output
                        .getPOStore(), output.getAlias(), stats));
            } else {
                HJob hjob = new HJob(HJob.JOB_STATUS.FAILED, pigContext, output
                        .getPOStore(), output.getAlias(), stats);
                hjob.setException(js.getException());
                jobs.add(hjob);
            }
        }
    }
    return jobs;
}

开发者ID:sigmoidanalytics，项目名称:spork，代码行数:32，代码来源:PigServer.java

示例12: testDuplicateCounterName2

import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Test
public void testDuplicateCounterName2() throws Exception {

    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = filter A by a0 > 3;");
    w1.println("store A into 'output';");
    w1.println("store B into 'tmp/output';");
    w1.close();

    try {
        String[] args = { "-x", execType, PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));

        assertTrue(stats.isSuccessful());

        assertEquals(1, stats.getNumberJobs());
        List<OutputStats> outputs = stats.getOutputStats();
        assertEquals(2, outputs.size());
        for (OutputStats outstats : outputs) {
            if (outstats.getLocation().endsWith("tmp/output")) {
                assertEquals(2, outstats.getNumberRecords());
            } else {
                assertEquals(5, outstats.getNumberRecords());
            }
        }
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
        Util.deleteFile(cluster, "tmp/output");
    }
}

开发者ID:sigmoidanalytics，项目名称:spork，代码行数:33，代码来源:TestPigRunner.java

示例13: testEmptyFileCounter2

import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Test //PIG-1893
public void testEmptyFileCounter2() throws Exception {

    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = filter A by a0 < 0;");
    w1.println("store A into '" + OUTPUT_FILE + "';");
    w1.println("store B into 'output2';");
    w1.close();

    try {
        String[] args = { "-x", execType, PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));

        assertTrue(stats.isSuccessful());

        assertEquals(1, stats.getNumberJobs());
        List<OutputStats> outputs = stats.getOutputStats();
        assertEquals(2, outputs.size());
        for (OutputStats outstats : outputs) {
            if (outstats.getLocation().endsWith("output2")) {
                assertEquals(0, outstats.getNumberRecords());
            } else {
                assertEquals(5, outstats.getNumberRecords());
            }
        }
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
        Util.deleteFile(cluster, "output2");
    }
}

开发者ID:sigmoidanalytics，项目名称:spork，代码行数:33，代码来源:TestPigRunner.java

示例14: testDisablePigCounters

import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Test // PIG-2208: Restrict number of PIG generated Haddop counters
public void testDisablePigCounters() throws Exception {
    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("C = join A by a0, B by a0;");
    w1.println("store C into '" + OUTPUT_FILE + "';");
    w1.close();

    try {
        String[] args = {"-Dpig.disable.counter=true", "-x", execType, PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));

        assertTrue(stats.isSuccessful());

        assertEquals(1, stats.getNumberJobs());
        List<InputStats> inputs = stats.getInputStats();
        assertEquals(2, inputs.size());
        if (execType.equals("tez")) {
            assertEquals(5, inputs.get(0).getNumberRecords());
            assertEquals(5, inputs.get(1).getNumberRecords());
        } else {
            for (InputStats instats : inputs) {
                // the multi-input counters are disabled
                assertEquals(-1, instats.getNumberRecords());
            }
        }

        List<OutputStats> outputs = stats.getOutputStats();
        assertEquals(1, outputs.size());
        OutputStats outstats = outputs.get(0);
        assertEquals(9, outstats.getNumberRecords());
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
    }
}

开发者ID:sigmoidanalytics，项目名称:spork，代码行数:38，代码来源:TestPigRunner.java

示例15: testDisablePigCounters2

import org.apache.pig.tools.pigstats.OutputStats; //导入依赖的package包/类
@Test // PIG-2208: Restrict number of PIG generated Haddop counters
public void testDisablePigCounters2() throws Exception {

    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = filter A by a0 > 3;");
    w1.println("store A into 'output';");
    w1.println("store B into 'tmp/output';");
    w1.close();

    try {
        String[] args = { "-Dpig.disable.counter=true", "-x", execType, PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));

        assertTrue(stats.isSuccessful());

        assertEquals(1, stats.getNumberJobs());
        List<OutputStats> outputs = stats.getOutputStats();
        assertEquals(2, outputs.size());
        if (execType.equals("tez")) {
            assertEquals(outputs.get(0).getNumberRecords(), 5);
            assertEquals(outputs.get(1).getNumberRecords(), 2);
        } else {
            for (OutputStats outstats : outputs) {
                // the multi-output counters are disabled
                assertEquals(-1, outstats.getNumberRecords());
            }
        }

        List<InputStats> inputs = stats.getInputStats();
        assertEquals(1, inputs.size());
        InputStats instats = inputs.get(0);
        assertEquals(5, instats.getNumberRecords());
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
        Util.deleteFile(cluster, "tmp/output");
    }
}

开发者ID:sigmoidanalytics，项目名称:spork，代码行数:40，代码来源:TestPigRunner.java

注：本文中的org.apache.pig.tools.pigstats.OutputStats类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。