Java PigServer.openIterator方法代码示例

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test
public void testInlineMacro() throws Throwable {
    PigServer pig = new PigServer(ExecType.LOCAL);

    Storage.Data data = resetData(pig);
    data.set("some_path", "(l:chararray)", tuple("first row"), tuple("second row"));

    pig.registerQuery("DEFINE row_count(X) RETURNS Z { Y = group $X all; $Z = foreach Y generate COUNT($X); };");
    pig.registerQuery("a = load 'some_path' USING mock.Storage();");
    pig.registerQuery("b = row_count(a);");
    Iterator<Tuple> iter = pig.openIterator("b");

    assertEquals(2L, ((Long)iter.next().get(0)).longValue());

    pig.shutdown();
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test
public void testLoadFromPigServer() throws Exception {
    String filename = TestHelper.createTempFile(data, " ");
    PigServer pig = new PigServer(ExecType.LOCAL);
    filename = filename.replace("\\", "\\\\");
    pig.registerQuery("A = LOAD '" + filename + "' USING org.apache.pig.piggybank.storage.apachelog.CombinedLogLoader();");
    Iterator<?> it = pig.openIterator("A");

    int tupleCount = 0;

    while (it.hasNext()) {
        Tuple tuple = (Tuple) it.next();
        if (tuple == null)
            break;
        else {
            TestHelper.examineTuple(EXPECTED, tuple, tupleCount);
            tupleCount++;
        }
    }
    assertEquals(data.size(), tupleCount);
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test
public void testNullTupleCols() throws Exception {
    String inputFileName = "TestProject-testNullTupleCols-input.txt";
    String input[] = { "1\t(hello,world)", "2\t(good)", "3" };
    Util.createLocalInputFile(inputFileName, input);
    // PigStorage will return null as the value for the tuple field in the
    // second record since it does not comply with the schema and in the
    // third record since the field is absent
    String query = "a = load '" + inputFileName + "' as (i:int, " +
            "t:tuple(s1:chararray, s2:chararray));" +
            "b = foreach a generate t.s1, t.s2;";

    PigServer ps = new PigServer(ExecType.LOCAL);
    Util.registerMultiLineQuery(ps, query);
    Iterator<Tuple> it = ps.openIterator("b");
    Tuple[] expectedResults = new Tuple[] {
                    (Tuple)Util.getPigConstant("('hello', 'world')"),
                    (Tuple)Util.getPigConstant("(null, null)"),
                    (Tuple)Util.getPigConstant("(null, null)")
    };
    int i = 0;
    while (it.hasNext()) {
        assertEquals(expectedResults[i++], it.next());
    }
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test
// See PIG-4184
public void testUDFNullInput() throws Exception {
    PigServer pig = new PigServer(ExecType.LOCAL);
    File inputFile = Util.createInputFile("tmp", "", 
            new String[] {"\t", "2\t3"});
    pig.registerQuery("a = load '"
            + Util.generateURI(inputFile.toString(), pig.getPigContext())
            + "' as (i1:int, i2:int);");
    pig.registerQuery("b = foreach a generate " + IntToBool.class.getName() + "(i1);");

    Iterator<Tuple> iter = pig.openIterator("b");
    assertEquals(iter.next().toString(), "(false)");
    assertEquals(iter.next().toString(), "(true)");
    assertFalse(iter.hasNext());
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test  
public void testMapsideGroupParserNoSupportForMultipleInputs() throws IOException {
    pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());

    pigServer.registerQuery("A = LOAD '" + INPUT_FILE + "' as (id, name, grade);");
    pigServer.registerQuery("B = LOAD '" + INPUT_FILE + "' as (id, name, grade);");

    try {
        pigServer.registerQuery("C = group A by id, B by id using 'collected';");
        pigServer.openIterator( "C" );
        Assert.fail("Pig doesn't support multi-input collected group.");
    } catch (Exception e) {
        String msg = "pig script failed to validate: Collected group is only supported for single input";
        Assert.assertTrue( e.getMessage().contains( msg ) );
    }
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
/**
 * Test UNION ONSCHEMA on two inputs with no common columns
 * @throws IOException
 * @throws ParserException
 */
@Test
public void testUnionOnSchemaNoCommonCols() throws IOException, ParserException {
    PigServer pig = new PigServer(ExecType.LOCAL);
    String query =
        "  l1 = load '" + INP_FILE_2NUMS + "' as (i : int, j : int);"
        + "l2 = load '" + INP_FILE_2NUMS + "' as (x : long, y : float);"
        + "u = union onschema l1, l2;"
    ; 
    Util.registerMultiLineQuery(pig, query);
    Iterator<Tuple> it = pig.openIterator("u");
    
    List<Tuple> expectedRes = 
        Util.getTuplesFromConstantTupleStrings(
                new String[] {
                        "(1,2,null,null)",
                        "(5,3,null,null)",
                        "(null,null,1L,2.0F)",
                        "(null,null,5L,3.0F)"
                });
    Util.checkQueryOutputsAfterSort(it, expectedRes);

}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test
public void testMissingCols2() throws Exception {
    String inputFileName = "TestProject-testMissingCols2-input.txt";
    String input[] = { "1\t(hello,world)", "2\t(good,bye)" };
    Util.createLocalInputFile(inputFileName, input);
    // in the script, PigStorage will return a null for the tuple field
    // since it does not comply with the schema
    String query = "a = load '" + inputFileName + "' as (i:int, " +
            "t:tuple(s1:chararray, s2:chararray, s3:chararray));" +
            "b = foreach a generate t.(s2,s3);";

    PigServer ps = new PigServer(ExecType.LOCAL);
    Util.registerMultiLineQuery(ps, query);
    Iterator<Tuple> it = ps.openIterator("b");
    Tuple[] expectedResults = new Tuple[] {
                    (Tuple)Util.getPigConstant("((null, null))"),
                    (Tuple)Util.getPigConstant("((null, null))")
    };
    int i = 0;
    while (it.hasNext()) {
        assertEquals(expectedResults[i++], it.next());
    }
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test
public void testLocalExecution() throws Exception {
    Iterator<Tuple> expectedItr = generateExpectedResults(DateTimeZone
            .forOffsetMillis(DateTimeZone.forID("+08:00").getOffset(null)));
    Properties config = new Properties();
    config.setProperty("pig.datetime.default.tz", "+08:00");
    PigServer pig = new PigServer(Util.getLocalTestMode(), config);
    pig.registerQuery("a = load '"
            + Util.encodeEscape(Util.generateURI(tmpFile.toString(), pig.getPigContext()))
            + "' as (test:datetime);");
    pig.registerQuery("b = filter a by test < ToDate('1970-01-04T00:00:00.000');");
    Iterator<Tuple> actualItr = pig.openIterator("b");
    while (expectedItr.hasNext() && actualItr.hasNext()) {
        Tuple expectedTuple = expectedItr.next();
        Tuple actualTuple = actualItr.next();
        assertEquals(expectedTuple, actualTuple);
    }
    assertEquals(expectedItr.hasNext(), actualItr.hasNext());
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
private void runQueryTest(String script, MockQueryHandler queryHandler, int port) throws Exception {
    final String endpoint = "http://localhost:" + port;

    HttpServer server = HttpServer.create(new InetSocketAddress(port), 0);
    server.createContext("/", queryHandler);
    server.start();

    PigServer ps = setup(script, endpoint);

    Iterator<Tuple> recommendations = ps.openIterator("recommendations");
    while (recommendations.hasNext()) {
        Tuple tuple = recommendations.next();

        String userid = (String) tuple.get(0);
        Integer rank = (Integer) tuple.get(1);
        String docid = (String) tuple.get(2);
        Double relevance = (Double) tuple.get(3);
        String fieldId = (String) tuple.get(4);
        String fieldContent = (String) tuple.get(5);

        MockQueryHandler.MockQueryHit hit = queryHandler.getHit(userid, rank);
        assertEquals(docid, hit.id);
        assertEquals(relevance, hit.relevance, 1e-3);
        assertEquals(fieldId, hit.fieldId);
        assertEquals(fieldContent, hit.fieldContent);
    }

    if (server != null) {
        server.stop(0);
    }

}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Override
public void execute(Query query) {
    String queryName = query.name;
    String queryValue = query.value;
    Map<String, String> queryMetadata = query.getMetadata();
    String execType = Query.getKey(queryMetadata, METADATA_EXEC_TYPE_KEY).orElse(defaultExecType);
    String alias = Query.getKey(queryMetadata, METADATA_ALIAS_KEY).orElse(defaultOutputAlias);
    log.info("Running {} for alias {}: {}", queryName, alias, queryValue);
    try {
        PigServer server = getPigServer(execType);
        server.registerScript(new ByteArrayInputStream(queryValue.getBytes()));
        Iterator<Tuple> queryResults = server.openIterator(alias);
        Result result = query.createResults();
        // dumpSchema will also, unfortunately, print the schema to stdout.
        List<FieldDetail> metadata = getFieldDetails(server.dumpSchema(alias));
        populateColumns(metadata, result);
        while (queryResults.hasNext()) {
            populateRow(queryResults.next(), metadata, result);
        }
        server.shutdown();
    } catch (IOException ioe) {
        log.error("Problem with Pig query: {}\n{}", queryValue, ioe);
        query.setFailure(ioe.toString());
    } catch (Exception e) {
        log.error("Error occurred while processing Pig query: {}\n{}", queryValue, e);
        query.setFailure(e.toString());
    }
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
/**
 * Test UNION ONSCHEMA where a common column has additional 'namespace' part
 *  in the column name in both the inputs
 * @throws IOException
 * @throws ParserException
 */
@Test
public void testUnionOnSchemaScopedColumnNameBothInp2() throws IOException, ParserException {
    PigServer pig = new PigServer(ExecType.LOCAL);
    String query =
        "   l1 = load '" + INP_FILE_2NUMS + "' as (i : int, j : int); " 
        + " l2 = load '" + INP_FILE_2NUMS + "' as (i : int, x : chararray); " 
        + " cg1 = cogroup l1 by i, l2 by i; "
        + " f1 = foreach cg1 generate group as gkey, flatten(l1), flatten(l2); "
        + " cg2 = cogroup l2 by i, l1 by i; "
        + " f2 = foreach cg1 generate group as gkey, flatten(l2), flatten(l1); "
        + "u = union onschema f1, f2; " ; 
    Util.registerMultiLineQuery(pig, query);
            
    Schema sch = pig.dumpSchema("u");
    Schema expectedSch = 
        Utils.getSchemaFromString("gkey: int, l1::i: int, l1::j: int, l2::i: int, l2::x: chararray");
    assertEquals("Checking expected schema",sch, expectedSch);

    Iterator<Tuple> it = pig.openIterator("u");
    List<Tuple> expectedRes = 
        Util.getTuplesFromConstantTupleStrings(
                new String[] {
                        "(1,1,2,1,'2')",
                        "(5,5,3,5,'3')",
                        "(1,1,2,1,'2')",
                        "(5,5,3,5,'3')",
                });
    Util.checkQueryOutputsAfterSort(it, expectedRes);
    
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test
public void testSimpleLoad() throws IOException, ParserException {
    PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
    String[] input = {
            "[key#1,key2#2]",
            "[key#2]",
    };

    Util.createInputFile(FileSystem.getLocal(new Configuration()), tmpDirName + "/table_testSimpleLoad", input);

    String query =
        "  a = load '" + tmpDirName + "/table_testSimpleLoad' as (m:map[int]);";
    Util.registerMultiLineQuery(pig, query);
    Schema sch = pig.dumpSchema("a");
    assertEquals("Checking expected schema",sch.toString(), "{m: map[int]}");
    Iterator<Tuple> it = pig.openIterator("a");

    Assert.assertTrue(it.hasNext());
    Tuple t = it.next();
    Assert.assertTrue(t.size()==1);
    Assert.assertTrue(t.get(0) instanceof Map);
    Assert.assertTrue(((Map)t.get(0)).containsKey("key"));
    Assert.assertTrue(((Map)t.get(0)).containsKey("key2"));
    Assert.assertTrue(((Map)t.get(0)).get("key") instanceof Integer);
    Assert.assertTrue(((Map)t.get(0)).get("key").toString().equals("1"));
    Assert.assertTrue(((Map)t.get(0)).get("key2") instanceof Integer);
    Assert.assertTrue(((Map)t.get(0)).get("key2").toString().equals("2"));

    Assert.assertTrue(it.hasNext());
    t = it.next();
    Assert.assertTrue(((Map)t.get(0)).containsKey("key"));
    Assert.assertTrue(((Map)t.get(0)).get("key") instanceof Integer);
    Assert.assertTrue(((Map)t.get(0)).get("key").toString().equals("2"));

    Assert.assertFalse(it.hasNext());
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test
public void testComplexCast2() throws IOException, ParserException {
    PigServer pig = new PigServer(ExecType.LOCAL, new Properties());
    String[] input = {
            "[key#1,key2#2]",
    };

    Util.createInputFile(FileSystem.getLocal(new Configuration()), tmpDirName + "/testComplexCast2", input);

    String query = "a = load '" + tmpDirName + "/testComplexCast2' as (m:[int]);" +
        "b = foreach a generate ([long])m;";
    Util.registerMultiLineQuery(pig, query);
    Schema sch = pig.dumpSchema("b");
    assertEquals("Checking expected schema",sch.toString(), "{m: map[long]}");
    Iterator<Tuple> it = pig.openIterator("b");

    Assert.assertTrue(it.hasNext());
    Tuple t = it.next();
    Assert.assertTrue(t.size()==1);
    Assert.assertTrue(t.get(0) instanceof Map);
    Assert.assertTrue(((Map)t.get(0)).containsKey("key"));
    Assert.assertTrue(((Map)t.get(0)).containsKey("key2"));
    Assert.assertTrue(((Map)t.get(0)).get("key") instanceof Long);
    Assert.assertTrue(((Map)t.get(0)).get("key").toString().equals("1"));
    Assert.assertTrue(((Map)t.get(0)).get("key2") instanceof Long);
    Assert.assertTrue(((Map)t.get(0)).get("key2").toString().equals("2"));

    Assert.assertFalse(it.hasNext());
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test
public void testHelperEvalFunc() throws Exception {
    String pref="org.apache.pig.test.utils.HelperEvalFuncUtils$";
    String[][] UDF = {
        {pref + "BasicSUM", pref + "AccSUM", pref + "AlgSUM", "SUM"},
        {pref + "BasicCOUNT", pref + "AccCOUNT", pref + "AlgCOUNT", "COUNT"},
        {"BasLCWC", "AccLCWC", "AlgLCWC", "5*COUNT"}
    };
    String input = "udf_test_helper_eval_func.txt";
    Util.createLocalInputFile(input, new String[]{"1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n11\n12\n13\n14\n15"});
    for (String[] udfs : UDF) {
        for (int i = 0; i < udfs.length - 1; i++) {
            String query = "DEFINE BasLCWC " + pref + "BasicLongCountWithConstructor('5');";
            query += "DEFINE AccLCWC " + pref +" AccLongCountWithConstructor('5');";
            query += "DEFINE AlgLCWC " + pref + "AlgLongCountWithConstructor('5');";
            query += "A = load '" + input + "' as (x:int);";
            query += "B = foreach (group A all) generate ";
            for (String s : Arrays.copyOfRange(udfs, i, udfs.length - 1)) {
                query += s + "(A),";
            }
            query += udfs[udfs.length - 1] + "(A);";
            PigServer pigServer = new PigServer(ExecType.LOCAL);
            pigServer.registerQuery(query);
            Iterator<Tuple> it = pigServer.openIterator("B");
            while (it.hasNext()) {
                Tuple t = it.next();
                Long val = (Long)t.get(0);
                for (int j = 1; j < i; j++) {
                    assertEquals(val, t.get(j));
                }
            }
        }
    }
}
 

import org.apache.pig.PigServer; //导入方法依赖的package包/类
@Test
public void testMultiSplits() throws Exception{

    PigServer pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
    pigServer.registerQuery("A = LOAD '" + INPUT_FILE1 + "," + INPUT_FILE4 + "' using "+ DummyCollectableLoader.class.getName() +"() as (c1:chararray,c2:int);");
    pigServer.registerQuery("B = LOAD '" + INPUT_FILE5 + "' using "+ DummyIndexableLoader.class.getName()   +"() as (c1:chararray,c2:int);");

    DataBag dbMergeCogrp = BagFactory.getInstance().newDefaultBag();

    pigServer.registerQuery("C = cogroup A by c1, B by c1 using 'merge';");
    Iterator<Tuple> iter = pigServer.openIterator("C");


    while(iter.hasNext()) {
        Tuple t = iter.next();
        dbMergeCogrp.add(t);
    }


    String[] results = new String[]{
            "(4,{(4,1),(4,2),(4,3)},{(4,1),(4,2),(4,3)})",
            "(5,{(5,2),(5,1),(5,3)},{(5,1),(5,2),(5,3)})",
            "(6,{(6,1),(6,2),(6,3)},{})",
            "(7,{(7,1),(7,2),(7,3)},{})",
            "(8,{(8,1),(8,2),(8,3)},{})",
            "(9,{(9,1),(9,2),(9,3)},{})",
            "(1,{(1,1),(1,2),(1,3)},{})",
            "(2,{(2,1),(2,2),(2,3)},{})",
            "(3,{(3,3),(3,2),(3,1)},{(3,1),(3,2),(3,3)})"
    };

    assertEquals(9, dbMergeCogrp.size());
    Iterator<Tuple> itr = dbMergeCogrp.iterator();
    for(int i=0; i<9; i++){
        assertEquals(itr.next().toString(), results[i]);   
    }
    assertFalse(itr.hasNext());
}