Java PigStats.getInputStats方法代码示例

本文整理汇总了Java中org.apache.pig.tools.pigstats.PigStats.getInputStats方法的典型用法代码示例。如果您正苦于以下问题：Java PigStats.getInputStats方法的具体用法？Java PigStats.getInputStats怎么用？Java PigStats.getInputStats使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.pig.tools.pigstats.PigStats的用法示例。

在下文中一共展示了PigStats.getInputStats方法的10个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: testLongCounterName

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
@Test
public void testLongCounterName() throws Exception {
    // Pig now restricts the string size of its counter name to less than 64 characters.
    PrintWriter w = new PrintWriter(new FileWriter("myinputfile"));
    w.println("1\t2\t3");
    w.println("5\t3\t4");
    w.println("3\t4\t5");
    w.println("5\t6\t7");
    w.println("3\t7\t8");
    w.close();
    String longfilename = "longlonglonglonglonglonglonglonglonglonglonglongfilefilefilename";
    Util.copyFromLocalToCluster(cluster, "myinputfile", longfilename);
    
    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = load '" + longfilename + "' as (a0:int, a1:int, a2:int);");
    w1.println("C = join A by a0, B by a0;");
    w1.println("store C into '" + OUTPUT_FILE + "';");
    w1.close();
    
    try {
        String[] args = { PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener());
 
        assertTrue(stats.isSuccessful());
        
        assertEquals(1, stats.getNumberJobs());
        List<InputStats> inputs = stats.getInputStats();
        assertEquals(2, inputs.size());
        for (InputStats instats : inputs) {
            assertEquals(5, instats.getNumberRecords());
        }
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
    }
}

开发者ID:sigmoidanalytics，项目名称:spork-streaming，代码行数:38，代码来源:TestPigRunner.java

示例2: testDuplicateCounterName

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
@Test
public void testDuplicateCounterName() throws Exception {
    // Pig now restricts the string size of its counter name to less than 64 characters.
    PrintWriter w = new PrintWriter(new FileWriter("myinputfile"));
    w.println("1\t2\t3");
    w.println("5\t3\t4");
    w.close();
    String samefilename = "tmp/input";
    Util.copyFromLocalToCluster(cluster, "myinputfile", samefilename);
    
    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = load '" + samefilename + "' as (a0:int, a1:int, a2:int);");
    w1.println("C = join A by a0, B by a0;");
    w1.println("store C into '" + OUTPUT_FILE + "';");
    w1.close();
    
    try {
        String[] args = { PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener());
 
        assertTrue(stats.isSuccessful());
        
        assertEquals(1, stats.getNumberJobs());
        List<InputStats> inputs = stats.getInputStats();
        assertEquals(2, inputs.size());
        for (InputStats instats : inputs) {
            if (instats.getLocation().endsWith("tmp/input")) {
                assertEquals(2, instats.getNumberRecords());
            } else {
                assertEquals(5, instats.getNumberRecords());
            }
        }
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
    }
}

开发者ID:sigmoidanalytics，项目名称:spork-streaming，代码行数:39，代码来源:TestPigRunner.java

示例3: testEmptyFileCounter

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
@Test //PIG-1893
public void testEmptyFileCounter() throws Exception {
  
    PrintWriter w = new PrintWriter(new FileWriter("myinputfile"));
    w.close();
     
    Util.copyFromLocalToCluster(cluster, "myinputfile", "1.txt");
    
    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = load '1.txt' as (a0:int, a1:int, a2:int);");
    w1.println("C = join A by a0, B by a0;");
    w1.println("store C into '" + OUTPUT_FILE + "';");
    w1.close();
    
    try {
        String[] args = { PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener());
 
        assertTrue(stats.isSuccessful());
        
        assertEquals(1, stats.getNumberJobs());
        List<InputStats> inputs = stats.getInputStats();
        assertEquals(2, inputs.size());
        for (InputStats instats : inputs) {
            if (instats.getLocation().endsWith("1.txt")) {
                assertEquals(0, instats.getNumberRecords());
            } else {
                assertEquals(5, instats.getNumberRecords());
            }
        }
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
    }
}

开发者ID:sigmoidanalytics，项目名称:spork-streaming，代码行数:37，代码来源:TestPigRunner.java

示例4: testDisablePigCounters

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
@Test // PIG-2208: Restrict number of PIG generated Haddop counters
public void testDisablePigCounters() throws Exception {        
    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("C = join A by a0, B by a0;");
    w1.println("store C into '" + OUTPUT_FILE + "';");
    w1.close();
    
    try {
        String[] args = { "-Dpig.disable.counter=true", PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener());
 
        assertTrue(stats.isSuccessful());
        
        assertEquals(1, stats.getNumberJobs());
        List<InputStats> inputs = stats.getInputStats();
        assertEquals(2, inputs.size());
        for (InputStats instats : inputs) {
            // the multi-input counters are disabled
            assertEquals(-1, instats.getNumberRecords());
        }
        
        List<OutputStats> outputs = stats.getOutputStats();
        assertEquals(1, outputs.size());
        OutputStats outstats = outputs.get(0);
        assertEquals(9, outstats.getNumberRecords());
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
    }
}

开发者ID:sigmoidanalytics，项目名称:spork-streaming，代码行数:33，代码来源:TestPigRunner.java

示例5: testDisablePigCounters2

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
@Test // PIG-2208: Restrict number of PIG generated Haddop counters
public void testDisablePigCounters2() throws Exception {
    
    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = filter A by a0 > 3;");
    w1.println("store A into 'output';");
    w1.println("store B into 'tmp/output';");
    w1.close();
    
    try {
        String[] args = { "-Dpig.disable.counter=true", PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener());
 
        assertTrue(stats.isSuccessful());
        
        assertEquals(1, stats.getNumberJobs());
        List<OutputStats> outputs = stats.getOutputStats();
        assertEquals(2, outputs.size());
        for (OutputStats outstats : outputs) {
            // the multi-output counters are disabled
            assertEquals(-1, outstats.getNumberRecords());
        }
        
        List<InputStats> inputs = stats.getInputStats();
        assertEquals(1, inputs.size());
        InputStats instats = inputs.get(0);
        assertEquals(5, instats.getNumberRecords());
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
        Util.deleteFile(cluster, "tmp/output");
    }
}

开发者ID:sigmoidanalytics，项目名称:spork-streaming，代码行数:35，代码来源:TestPigRunner.java

示例6: testLongCounterName

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
@Test
public void testLongCounterName() throws Exception {
    // Pig now restricts the string size of its counter name to less than 64 characters.
    PrintWriter w = new PrintWriter(new FileWriter("myinputfile"));
    w.println("1\t2\t3");
    w.println("5\t3\t4");
    w.println("3\t4\t5");
    w.println("5\t6\t7");
    w.println("3\t7\t8");
    w.close();
    String longfilename = "longlonglonglonglonglonglonglonglonglonglonglongfilefilefilename";
    Util.copyFromLocalToCluster(cluster, "myinputfile", longfilename);

    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = load '" + longfilename + "' as (a0:int, a1:int, a2:int);");
    w1.println("C = join A by a0, B by a0;");
    w1.println("store C into '" + OUTPUT_FILE + "';");
    w1.close();

    try {
        String[] args = { "-x", execType, PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));

        assertTrue(stats.isSuccessful());

        assertEquals(1, stats.getNumberJobs());
        List<InputStats> inputs = stats.getInputStats();
        assertEquals(2, inputs.size());
        for (InputStats instats : inputs) {
            assertEquals(5, instats.getNumberRecords());
        }
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
    }
}

开发者ID:sigmoidanalytics，项目名称:spork，代码行数:38，代码来源:TestPigRunner.java

示例7: testDuplicateCounterName

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
@Test
public void testDuplicateCounterName() throws Exception {
    // Pig now restricts the string size of its counter name to less than 64 characters.
    PrintWriter w = new PrintWriter(new FileWriter("myinputfile"));
    w.println("1\t2\t3");
    w.println("5\t3\t4");
    w.close();
    String samefilename = "tmp/input";
    Util.copyFromLocalToCluster(cluster, "myinputfile", samefilename);

    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = load '" + samefilename + "' as (a0:int, a1:int, a2:int);");
    w1.println("C = join A by a0, B by a0;");
    w1.println("store C into '" + OUTPUT_FILE + "';");
    w1.close();

    try {
        String[] args = { "-x", execType, PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));

        assertTrue(stats.isSuccessful());

        assertEquals(1, stats.getNumberJobs());
        List<InputStats> inputs = stats.getInputStats();
        assertEquals(2, inputs.size());
        for (InputStats instats : inputs) {
            if (instats.getLocation().endsWith("tmp/input")) {
                assertEquals(2, instats.getNumberRecords());
            } else {
                assertEquals(5, instats.getNumberRecords());
            }
        }
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
    }
}

开发者ID:sigmoidanalytics，项目名称:spork，代码行数:39，代码来源:TestPigRunner.java

示例8: testEmptyFileCounter

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
@Test //PIG-1893
public void testEmptyFileCounter() throws Exception {

    PrintWriter w = new PrintWriter(new FileWriter("myinputfile"));
    w.close();

    Util.copyFromLocalToCluster(cluster, "myinputfile", "1.txt");

    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = load '1.txt' as (a0:int, a1:int, a2:int);");
    w1.println("C = join A by a0, B by a0;");
    w1.println("store C into '" + OUTPUT_FILE + "';");
    w1.close();

    try {
        String[] args = { "-x", execType, PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));

        assertTrue(stats.isSuccessful());

        assertEquals(1, stats.getNumberJobs());
        List<InputStats> inputs = stats.getInputStats();
        assertEquals(2, inputs.size());
        for (InputStats instats : inputs) {
            if (instats.getLocation().endsWith("1.txt")) {
                assertEquals(0, instats.getNumberRecords());
            } else {
                assertEquals(5, instats.getNumberRecords());
            }
        }
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
    }
}

开发者ID:sigmoidanalytics，项目名称:spork，代码行数:37，代码来源:TestPigRunner.java

示例9: testDisablePigCounters

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
@Test // PIG-2208: Restrict number of PIG generated Haddop counters
public void testDisablePigCounters() throws Exception {
    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("C = join A by a0, B by a0;");
    w1.println("store C into '" + OUTPUT_FILE + "';");
    w1.close();

    try {
        String[] args = {"-Dpig.disable.counter=true", "-x", execType, PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));

        assertTrue(stats.isSuccessful());

        assertEquals(1, stats.getNumberJobs());
        List<InputStats> inputs = stats.getInputStats();
        assertEquals(2, inputs.size());
        if (execType.equals("tez")) {
            assertEquals(5, inputs.get(0).getNumberRecords());
            assertEquals(5, inputs.get(1).getNumberRecords());
        } else {
            for (InputStats instats : inputs) {
                // the multi-input counters are disabled
                assertEquals(-1, instats.getNumberRecords());
            }
        }

        List<OutputStats> outputs = stats.getOutputStats();
        assertEquals(1, outputs.size());
        OutputStats outstats = outputs.get(0);
        assertEquals(9, outstats.getNumberRecords());
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
    }
}

开发者ID:sigmoidanalytics，项目名称:spork，代码行数:38，代码来源:TestPigRunner.java

示例10: testDisablePigCounters2

import org.apache.pig.tools.pigstats.PigStats; //导入方法依赖的package包/类
@Test // PIG-2208: Restrict number of PIG generated Haddop counters
public void testDisablePigCounters2() throws Exception {

    PrintWriter w1 = new PrintWriter(new FileWriter(PIG_FILE));
    w1.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
    w1.println("B = filter A by a0 > 3;");
    w1.println("store A into 'output';");
    w1.println("store B into 'tmp/output';");
    w1.close();

    try {
        String[] args = { "-Dpig.disable.counter=true", "-x", execType, PIG_FILE };
        PigStats stats = PigRunner.run(args, new TestNotificationListener(execType));

        assertTrue(stats.isSuccessful());

        assertEquals(1, stats.getNumberJobs());
        List<OutputStats> outputs = stats.getOutputStats();
        assertEquals(2, outputs.size());
        if (execType.equals("tez")) {
            assertEquals(outputs.get(0).getNumberRecords(), 5);
            assertEquals(outputs.get(1).getNumberRecords(), 2);
        } else {
            for (OutputStats outstats : outputs) {
                // the multi-output counters are disabled
                assertEquals(-1, outstats.getNumberRecords());
            }
        }

        List<InputStats> inputs = stats.getInputStats();
        assertEquals(1, inputs.size());
        InputStats instats = inputs.get(0);
        assertEquals(5, instats.getNumberRecords());
    } finally {
        new File(PIG_FILE).delete();
        Util.deleteFile(cluster, OUTPUT_FILE);
        Util.deleteFile(cluster, "tmp/output");
    }
}

开发者ID:sigmoidanalytics，项目名称:spork，代码行数:40，代码来源:TestPigRunner.java

注：本文中的org.apache.pig.tools.pigstats.PigStats.getInputStats方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。