Java LogicalPlan.explain方法代码示例

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
/**
 * Test that SAMPLE doesn't get pushed up (see PIG-2014)
 */
@Test
public void testSample() throws Exception {
    String query = "A = LOAD 'file.txt' AS (name, cuisines:bag{ t : ( cuisine ) } );" +
    "B = GROUP A by name;" +
    "C = FOREACH B GENERATE group, A;" +
    "D = SAMPLE C 0.1 ; " +
    "E = STORE D INTO 'empty';";
    // expect loload -> foreach -> cogroup -> filter
    LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
    newLogicalPlan.explain(System.out, "text", true);

    Operator load = newLogicalPlan.getSources().get( 0 );
    Assert.assertTrue( load instanceof LOLoad );
    Operator fe1 = newLogicalPlan.getSuccessors( load ).get( 0 );
    Assert.assertTrue( fe1 instanceof LOForEach );
    Operator cg = newLogicalPlan.getSuccessors( fe1 ).get( 0 );
    Assert.assertTrue( cg instanceof LOCogroup );
    Operator fe2 = newLogicalPlan.getSuccessors( cg ).get( 0 );
    Assert.assertTrue( fe1 instanceof LOForEach );
    Operator filter = newLogicalPlan.getSuccessors( fe2 ).get( 0 );
    Assert.assertTrue( filter instanceof LOFilter );
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
/**
 * Test that SAMPLE doesn't get pushed up over Distinct (see PIG-2137)
 */
@Test
public void testSampleDistinct() throws Exception {
    String query = "A = LOAD 'file.txt' AS (name, cuisines:bag{ t : ( cuisine ) } );" +
    "B = DISTINCT A;" +
    "C = SAMPLE B 0.1 ; " +
    "D = STORE C INTO 'empty';";
    // expect loload -> foreach -> distinct -> filter
    LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
    newLogicalPlan.explain(System.out, "text", true);

    Operator load = newLogicalPlan.getSources().get( 0 );
    Assert.assertTrue( load instanceof LOLoad );
    Operator fe1 = newLogicalPlan.getSuccessors( load ).get( 0 );
    Assert.assertTrue( fe1 instanceof LOForEach );
    Operator dist = newLogicalPlan.getSuccessors( fe1 ).get( 0 );
    Assert.assertTrue( dist instanceof LODistinct );
    Operator filter = newLogicalPlan.getSuccessors( dist ).get( 0 );
    Assert.assertTrue( filter instanceof LOFilter );
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
/**
 * Test that deterministic filter gets get pushed up over Distinct (see PIG-2137)
 */
@Test
public void testFilterAfterDistinct() throws Exception {
    String query = "A = LOAD 'file.txt' AS (name : chararray, cuisines:bag{ t : ( cuisine ) } );" +
    "B = DISTINCT A;" +
    "C = filter B by SIZE(name) > 10;" +
    "D = STORE C INTO 'long_name';";
    // filter should be pushed above distinct,
    //ie expect - loload -> foreach -> filter -> distinct 
    LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
    newLogicalPlan.explain(System.out, "text", true);

    Operator load = newLogicalPlan.getSources().get( 0 );
    Assert.assertTrue( load instanceof LOLoad );
    Operator fe1 = newLogicalPlan.getSuccessors( load ).get( 0 );
    Assert.assertTrue( fe1 instanceof LOForEach );
    Operator filter = newLogicalPlan.getSuccessors(fe1).get( 0 );
    Assert.assertTrue( filter instanceof LOFilter );
   
    Operator dist = newLogicalPlan.getSuccessors(filter).get( 0 );
    Assert.assertTrue( dist instanceof LODistinct );
    
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
private LogicalPlan createAndProcessLPlan(String query) throws FrontendException {
    LogicalPlan plan = generateLogicalPlan(query);

   // new ProjectStarExpander(plan).visit();

    new ColumnAliasConversionVisitor( plan ).visit();
   // new ScalarVisitor( plan, pigContext ).visit();

    CompilationMessageCollector collector = new CompilationMessageCollector();
    new TypeCheckingRelVisitor( plan, collector).visit();
    new UnionOnSchemaSetter( plan ).visit();
    new CastLineageSetter(plan, collector).visit();

    printMessageCollector(collector);
    plan.explain(System.out, "text", true);

    if (collector.hasError()) {
        throw new AssertionError("Expect no  error");
    }
    return plan;
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
/**
 * Non-deterministic filters should not be pushed up (see PIG-2014).
 * In the example below, if Filter gets pushed above flatten, we might remove
 * whole bags of cuisines of random gets pushed up, while the intent is to sample from each bag.
 * @throws Exception
 */
@Test
public void testNondeterministicFilter() throws Exception {
    String query = "A =LOAD 'file.txt' AS (name, cuisines:bag{ t : ( cuisine ) }, num:int );" +
    "B = FOREACH A GENERATE name, flatten(cuisines), num;" +
    "C = FILTER B BY RANDOM(num) > 5;" +
    "D = STORE C INTO 'empty';" ;

    LogicalPlan newLogicalPlan = buildPlan( query );

    newLogicalPlan.explain(System.out, "text", true);

    // Expect Filter to not be pushed, so it should be load->foreach-> filter
    Operator load = newLogicalPlan.getSources().get( 0 );
    Assert.assertTrue( load instanceof LOLoad );
    Operator fe1 = newLogicalPlan.getSuccessors( load ).get( 0 );
    Assert.assertTrue( fe1 instanceof LOForEach );
    Operator fe2 = newLogicalPlan.getSuccessors( fe1 ).get( 0 );
    Assert.assertTrue( fe2 instanceof LOForEach );
    Operator filter = newLogicalPlan.getSuccessors( fe2 ).get( 0 );
    Assert.assertTrue( filter instanceof LOFilter );
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
/**
 * Test that filter cannot get pushed up over nested Distinct (see PIG-3347)
 */
@Test
public void testFilterAfterNestedDistinct() throws Exception {
    String query = "a = LOAD 'file.txt';" +
        "a_group = group a by $0;" +
        "b = foreach a_group { a_distinct = distinct a.$0;generate group, a_distinct;}" +
        "c = filter b by SIZE(a_distinct) == 1;" +
        "store c into 'empty';";

    // filter should not be pushed above nested distinct,
    //ie expect - loload -> locogroup -> foreach -> filter
    LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
    newLogicalPlan.explain(System.out, "text", true);

    Operator load = newLogicalPlan.getSources().get( 0 );
    Assert.assertTrue( load instanceof LOLoad );
    Operator cogroup = newLogicalPlan.getSuccessors( load ).get( 0 );
    Assert.assertTrue( cogroup instanceof LOCogroup );
    Operator foreach = newLogicalPlan.getSuccessors(cogroup).get( 0 );
    Assert.assertTrue( foreach instanceof LOForEach );
    Operator filter = newLogicalPlan.getSuccessors(foreach).get( 0 );
    Assert.assertTrue( filter instanceof LOFilter );
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
/**
 * Test that filter cannot get pushed up over nested Limit (see PIG-3347)
 */
@Test
public void testFilterAfterNestedLimit() throws Exception {
    String query = "a = LOAD 'file.txt';" +
        "a_group = group a by $0;" +
        "b = foreach a_group { a_limit = limit a.$0 5;generate group, a_limit;}" +
        "c = filter b by SIZE(a_limit) == 1;" +
        "store c into 'empty';";

    // filter should not be pushed above nested distinct,
    //ie expect - loload -> locogroup -> foreach -> filter
    LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
    newLogicalPlan.explain(System.out, "text", true);

    Operator load = newLogicalPlan.getSources().get( 0 );
    Assert.assertTrue( load instanceof LOLoad );
    Operator cogroup = newLogicalPlan.getSuccessors( load ).get( 0 );
    Assert.assertTrue( cogroup instanceof LOCogroup );
    Operator foreach = newLogicalPlan.getSuccessors(cogroup).get( 0 );
    Assert.assertTrue( foreach instanceof LOForEach );
    Operator filter = newLogicalPlan.getSuccessors(foreach).get( 0 );
    Assert.assertTrue( filter instanceof LOFilter );
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
/**
 * Test that filter cannot get pushed up over nested Filter (see PIG-3347)
 */
@Test
public void testFilterAfterNestedFilter() throws Exception {
    String query = "a = LOAD 'file.txt';" +
        "a_group = group a by $0;" +
        "b = foreach a_group { a_filter = filter a by $0 == 1;generate group, a_filter;}" +
        "c = filter b by SIZE(a_filter) == 1;" +
        "store c into 'empty';";

    // filter should not be pushed above nested distinct,
    //ie expect - loload -> locogroup -> foreach -> filter
    LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
    newLogicalPlan.explain(System.out, "text", true);

    Operator load = newLogicalPlan.getSources().get( 0 );
    Assert.assertTrue( load instanceof LOLoad );
    Operator cogroup = newLogicalPlan.getSuccessors( load ).get( 0 );
    Assert.assertTrue( cogroup instanceof LOCogroup );
    Operator foreach = newLogicalPlan.getSuccessors(cogroup).get( 0 );
    Assert.assertTrue( foreach instanceof LOForEach );
    Operator filter = newLogicalPlan.getSuccessors(foreach).get( 0 );
    Assert.assertTrue( filter instanceof LOFilter );
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
/**
 * Test that filter does not get blocked for PushUpFilter/FilterAboveForeach
 * by an unrelated nested filter (see PIG-3347)
 */
@Test
public void testFilterAfterUnrelatedNestedFilter() throws Exception {
    String query = "a = LOAD 'file.txt' as (a0:int, a1_bag:bag{(X:int)}, a2_bag:bag{(Y:int)});" +
        "b = foreach a { a1_filter = filter a1_bag by X == 1; generate a0, a1_filter, a2_bag;}" +
        "c = filter b by SIZE(a2_bag) == 1;" +
        "store c into 'empty';";

    // filter should be pushed above nested filter,
    //ie expect - loload -> locogroup -> foreach -> filter
    LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
    newLogicalPlan.explain(System.out, "text", true);

    Operator load = newLogicalPlan.getSources().get( 0 );
    Assert.assertTrue( load instanceof LOLoad );
    Operator foreach1 = newLogicalPlan.getSuccessors(load).get( 0 );
    Assert.assertTrue( foreach1 instanceof LOForEach );
    Operator filter = newLogicalPlan.getSuccessors( foreach1 ).get( 0 );
    Assert.assertTrue( filter instanceof LOFilter );
    Operator foreach2 = newLogicalPlan.getSuccessors(filter).get( 0 );
    Assert.assertTrue( foreach2 instanceof LOForEach );
}