Java LogicalPlan类代码示例

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testSimpleNotPossible() throws Exception {
    String query = "A =LOAD 'file.txt' AS (name, cuisines:bag{ t : ( cuisine ) } );" +
            "B = FOREACH A GENERATE name, flatten(cuisines) as cuisines;" +
            "C = FILTER B BY cuisines == 'pizza';" +
            "D = STORE C INTO 'empty';" ;  
    LogicalPlan newLogicalPlan = buildPlan( query );
    
    Operator load = newLogicalPlan.getSources().get( 0 );
    Assert.assertTrue( load instanceof LOLoad );
    Operator fe1  = newLogicalPlan.getSuccessors( load ).get( 0 );
    Assert.assertTrue( fe1 instanceof LOForEach );
    Operator fe2 = newLogicalPlan.getSuccessors( fe1 ).get( 0 );
    Assert.assertTrue( fe2 instanceof LOForEach );
    Operator filter = newLogicalPlan.getSuccessors( fe2 ).get( 0 );
    Assert.assertTrue( filter instanceof LOFilter );
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testSimple() throws Exception {
    String query = "A =LOAD 'file.txt' AS (name, cuisines:bag{ t : ( cuisine ) } );" +
    "B = FOREACH A GENERATE name, flatten(cuisines);" +
    "C = FILTER B BY name == 'joe';" +
    "D = STORE C INTO 'empty';" ;  
    LogicalPlan newLogicalPlan = buildPlan( query );

    Operator load = newLogicalPlan.getSources().get( 0 );
    Assert.assertTrue( load instanceof LOLoad );
    Operator filter = newLogicalPlan.getSuccessors( load ).get( 0 );
    Assert.assertTrue( filter instanceof LOFilter );
    Operator fe1 = newLogicalPlan.getSuccessors( filter ).get( 0 );
    Assert.assertTrue( fe1 instanceof LOForEach );
    Operator fe2 = newLogicalPlan.getSuccessors( fe1 ).get( 0 );
    Assert.assertTrue( fe2 instanceof LOForEach );
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testFilterFilter() throws Exception {
    String query = "A = load 'myfile' as (name, age, gpa);" +
        "B = filter A by $0 != 'name';" +
        "C = filter B by $1 < 18;" +
        "D = STORE C into 'dummy';";
    
    LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );

    Operator op = newLogicalPlan.getSources().get( 0 );
    Assert.assertTrue(  op instanceof LOLoad );
    op = newLogicalPlan.getSuccessors(op).get( 0 );
    Assert.assertTrue(  op instanceof LOForEach );
    op = newLogicalPlan.getSuccessors(op).get( 0 );
    Assert.assertTrue(  op instanceof LOFilter );
    Assert.assertTrue( ((LOFilter)op).getAlias().equals( "B" ) );
    op = newLogicalPlan.getSuccessors(op).get( 0 );
    Assert.assertTrue(  op instanceof LOFilter );
    Assert.assertTrue( ((LOFilter)op).getAlias().equals( "C" ) );
    op = newLogicalPlan.getSuccessors(op).get( 0 );
    Assert.assertTrue(  op instanceof LOStore );
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public void visit(LOSplit split) throws FrontendException {
    List<Operator> branchOutputs = split.getPlan().getSuccessors(split);
    for (int i=0;i<branchOutputs.size();i++) {
        Operator branchOutput = branchOutputs.get(i);
        Set<Long> branchOutputUids = (Set<Long>)branchOutput.getAnnotation(ColumnPruneHelper.INPUTUIDS);

        if (branchOutputUids!=null) {
            Set<Integer> columnsToDrop = new HashSet<Integer>();

            for (int j=0;j<split.getSchema().size();j++) {
                if (!branchOutputUids.contains(split.getSchema().getField(j).uid))
                    columnsToDrop.add(j);
            }

            if (!columnsToDrop.isEmpty()) {
                LOForEach foreach = Util.addForEachAfter((LogicalPlan)split.getPlan(), split, i, columnsToDrop);
                foreach.getSchema();
            }
        }
    }
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
private void removeSubTree(LogicalRelationalOperator op) throws FrontendException {
    LogicalPlan p = (LogicalPlan)op.getPlan();
    List<Operator> ll = p.getPredecessors(op);
    if (ll != null) {
        for(Operator pred: ll.toArray(new Operator[ll.size()])) {
            removeSubTree((LogicalRelationalOperator)pred);
        }
    }

    if (p.getSuccessors(op) != null) {
        Operator[] succs = p.getSuccessors(op).toArray(new Operator[0]);
        for(Operator s: succs) {
            p.disconnect(op, s);
        }
    }

    p.remove(op);
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
/**
 * No fields are used in filter condition at all.
 * @throws Exception 
 */
@Test
public void test3() throws Exception {
    String query = "A =LOAD 'file.txt' AS (a:(u,v), b, c);" +
    "B = FOREACH A GENERATE $0, b;" +
    "C = FILTER B BY 8 > 5;" +
    "STORE C INTO 'empty';";  
    LogicalPlan newLogicalPlan = buildPlan( query );

    Operator load = newLogicalPlan.getSources().get( 0 );
    Assert.assertTrue( load instanceof LOLoad );
    Operator filter = newLogicalPlan.getSuccessors( load ).get( 0 );
    Assert.assertTrue( filter instanceof LOFilter );
    Operator fe1 = newLogicalPlan.getSuccessors( filter ).get( 0 );
    Assert.assertTrue( fe1 instanceof LOForEach );
    Operator fe2 = newLogicalPlan.getSuccessors( fe1 ).get( 0 );
    Assert.assertTrue( fe2 instanceof LOForEach );
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
private void checkWarning(String query, String warnMsg) throws FrontendException {
    LogicalPlan plan = generateLogicalPlan(query);
    new ColumnAliasConversionVisitor( plan ).visit();
    // validate
    CompilationMessageCollector collector = new CompilationMessageCollector();
    TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
    typeChecker.visit();
    new CastLineageSetter(plan, collector).visit();

    printMessageCollector(collector);

    boolean isWarningSeen = false;
    assertTrue("message collector has message", collector.hasMessage());

    for (Message msg : collector){
        if (msg.getMessageType() == MessageType.Warning
                && msg.getMessage().contains(warnMsg)){
            isWarningSeen = true;
        }
    }

    assertTrue("Expected warning is seen", isWarningSeen);
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
/**
 * Test that filter cannot get pushed up over nested Limit (see PIG-3347)
 */
@Test
public void testFilterAfterNestedLimit() throws Exception {
    String query = "a = LOAD 'file.txt';" +
        "a_group = group a by $0;" +
        "b = foreach a_group { a_limit = limit a.$0 5;generate group, a_limit;}" +
        "c = filter b by SIZE(a_limit) == 1;" +
        "store c into 'empty';";

    // filter should not be pushed above nested distinct,
    //ie expect - loload -> locogroup -> foreach -> filter
    LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
    newLogicalPlan.explain(System.out, "text", true);

    Operator load = newLogicalPlan.getSources().get( 0 );
    Assert.assertTrue( load instanceof LOLoad );
    Operator cogroup = newLogicalPlan.getSuccessors( load ).get( 0 );
    Assert.assertTrue( cogroup instanceof LOCogroup );
    Operator foreach = newLogicalPlan.getSuccessors(cogroup).get( 0 );
    Assert.assertTrue( foreach instanceof LOForEach );
    Operator filter = newLogicalPlan.getSuccessors(foreach).get( 0 );
    Assert.assertTrue( filter instanceof LOFilter );
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testBuildLoadOpWithDefaultFunc() throws Exception {
    String query = "a = load '1.txt';" +
            "store a into 'output';";
    LogicalPlan lp = buildPlan(query);
    FuncSpec funcSpec = getFirstLoadFuncSpec(lp);
    assertEquals("org.apache.pig.builtin.PigStorage", funcSpec.getClassName());
    
    // set default load func in config
    pigServer.getPigContext().getProperties().setProperty(PigConfiguration.PIG_DEFAULT_LOAD_FUNC, "org.apache.pig.test.PigStorageWithSchema");
    query = "a = load '1.txt';" +
            "store a into 'output';";
    lp = buildPlan(query);
    funcSpec = getFirstLoadFuncSpec(lp);
    assertEquals("org.apache.pig.test.PigStorageWithSchema", funcSpec.getClassName());    
    
    // unset default load func
    pigServer.getPigContext().getProperties().remove(PigConfiguration.PIG_DEFAULT_LOAD_FUNC);      
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testMultiQueryWithTwoStores() {

    System.out.println("===== multi-query with 2 stores =====");

    try {
        myPig.setBatchOn();

        myPig.registerQuery("a = load 'passwd' " +
                            "using PigStorage(':') as (uname:chararray, passwd:chararray, uid:int,gid:int);");
        myPig.registerQuery("b = filter a by uid > 5;");
        myPig.registerQuery("store b into '/tmp/output1';");
        myPig.registerQuery("c = group b by gid;");
        myPig.registerQuery("store c into '/tmp/output2';");

        LogicalPlan lp = checkLogicalPlan(1, 2, 5);

        PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 11);

        checkMRPlan(pp, 1, 1, 1);

    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail();
    } 
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testFilterConstantConditionGroupBy() throws Exception {
    String query = "A = load 'myfile' as (name, age, gpa);" +
        "B = group A by $0;" +
        "C = filter B by 1 == 1;" +
        "D = STORE C into 'dummy';";
    
    LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );

    Operator op = newLogicalPlan.getSources().get( 0 );
    Assert.assertTrue(  op instanceof LOLoad );
    op = newLogicalPlan.getSuccessors(op).get( 0 );
    Assert.assertTrue(  op instanceof LOForEach );
    op = newLogicalPlan.getSuccessors(op).get( 0 );
    Assert.assertTrue(  op instanceof LOFilter );
    op = newLogicalPlan.getSuccessors(op).get( 0 );
    Assert.assertTrue(  op instanceof LOCogroup );
    op = newLogicalPlan.getSuccessors(op).get( 0 );
    Assert.assertTrue(  op instanceof LOStore );
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testNotPossibleFilter() throws Exception {
    String query = "A =LOAD 'file.txt' AS (name, cuisines:bag{ t : ( cuisine ) } );" +
    "B = FOREACH A GENERATE name, flatten(cuisines);" +
    "C = FILTER B BY cuisine == 'French';" +
    "D = STORE C INTO 'empty';";  
    LogicalPlan newLogicalPlan = buildPlan( query );

    Operator load = newLogicalPlan.getSources().get( 0 );
    Assert.assertTrue( load instanceof LOLoad );
    Operator fe1 = newLogicalPlan.getSuccessors( load ).get( 0 );
    Assert.assertTrue( fe1 instanceof LOForEach );
    Operator fe2 = newLogicalPlan.getSuccessors( fe1 ).get( 0 );
    Assert.assertTrue( fe2 instanceof LOForEach );
    Operator filter = newLogicalPlan.getSuccessors( fe2 ).get( 0 );
    Assert.assertTrue( filter instanceof LOFilter );
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testFilterConstantConditionGroupBy() throws Exception {
    String query = "A = load 'myfile' as (name, age, gpa);" +
        "B = group A by $0;" +
        "C = filter B by 1 == 1;" +
        "D = STORE C into 'dummy';";
    
    LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );

    Operator op = newLogicalPlan.getSources().get( 0 );
    Assert.assertTrue(  op instanceof LOLoad );
    op = newLogicalPlan.getSuccessors(op).get( 0 );
    Assert.assertTrue(  op instanceof LOFilter );
    op = newLogicalPlan.getSuccessors(op).get( 0 );
    Assert.assertTrue(  op instanceof LOCogroup );
    op = newLogicalPlan.getSuccessors(op).get( 0 );
    Assert.assertTrue(  op instanceof LOStore );
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
/**
 * @return a dummy logical relational operator
 */
private LogicalRelationalOperator createDummyRelOpWithAlias() {
    class DummyRelOp extends LogicalRelationalOperator{
        DummyRelOp(){
            super("dummy", new LogicalPlan());
            this.alias = "dummy";
        }

        @Override
        public LogicalSchema getSchema() throws FrontendException {
                      return null;
        }

        @Override
        public void accept(PlanVisitor v) throws FrontendException {

        }

        @Override
        public boolean isEqual(Operator operator) throws FrontendException {
            return false;
        }

    }
    return new DummyRelOp();
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
public static void attachStorePlan(String scope, LogicalPlan lp, String fileName, String func, 
        Operator input, String alias, PigContext pigContext) throws FrontendException {
    func = func == null ? pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName()) : func;

    FuncSpec funcSpec = new FuncSpec( func );
    StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec( funcSpec );
    
    fileName = removeQuotes( fileName );
    FileSpec fileSpec = new FileSpec( fileName, funcSpec );
    String sig = alias + "_" + LogicalPlanBuilder.newOperatorKey(scope);
    stoFunc.setStoreFuncUDFContextSignature(sig);
    LOStore store = new LOStore(lp, fileSpec, stoFunc, sig);
    store.setAlias(alias);

    try {
        stoFunc.relToAbsPathForStoreLocation( fileName, getCurrentDir( pigContext ) );
    } catch (IOException ioe) {
        FrontendException e = new FrontendException(  ioe.getMessage(), ioe );
        throw e;
    }

    lp.add( store );
    lp.connect( input, store );
}