Java LogicalPlan.getOperators方法代码示例

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
public Map<LOForEach, Map<LogicalRelationalOperator, PhysicalOperator>> getForEachInnerLogToPhyMap(LogicalPlan plan) {
    Map<LOForEach, Map<LogicalRelationalOperator, PhysicalOperator>> result =
        new HashMap<LOForEach, Map<LogicalRelationalOperator, PhysicalOperator>>();
    Iterator<Operator> outerIter = plan.getOperators();
    while (outerIter.hasNext()) {
        Operator oper = outerIter.next();
        if (oper instanceof LOForEach) {
            LogicalPlan innerPlan = ((LOForEach) oper).getInnerPlan();
            Map<LogicalRelationalOperator, PhysicalOperator> innerOpMap = new HashMap<LogicalRelationalOperator, PhysicalOperator>();
            Iterator<Operator> innerIter = innerPlan.getOperators();
            while (innerIter.hasNext()) {
                Operator innerOper = innerIter.next();
                innerOpMap.put(((LogicalRelationalOperator)innerOper), newLogToPhyMap.get(innerOper));
            }
            result.put((LOForEach) oper, innerOpMap);
        }
    }
    return result;
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
/**
 * Test that pig sends correct partition column names in setPartitionFilter
 * when the user has a schema in the load statement which renames partition
 * columns - in this test case the filter only has conditions on partition
 * columns
 * @throws Exception
 */
@Test
public void testColNameMapping3() throws Exception {
    TestLoader.partFilter = null;
    String query = "a = load 'foo' using "
        + TestLoader.class.getName() +
        "('srcid:int, mrkt:chararray, dstid:int, name:chararray, age:int', " +
        "'srcid,mrkt,dstid,age') as (f1, f2, f3, f4, f5);" +
        "b = filter a by " +
        "(f5 >= 20 or f2 == 'us') and (f1 == 10 and f3 == 15);" +
        "store b into 'out';";

    LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );

    Assert.assertEquals("checking partition filter:",
            "(((age >= 20) or (mrkt == 'us')) and ((srcid == 10) and " +
            "(dstid == 15)))",
            TestLoader.partFilter.toString());
    Iterator<Operator> it = newLogicalPlan.getOperators();
    while( it.hasNext() ) {
     Assert.assertFalse("Checking that filter has been removed since it contained" +
             " only conditions on partition cols:",
             (it.next() instanceof LOFilter));
    }
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
/**
 * Find relational operators that are instance of class c
 * @param plan
 * @param c
 * @return
 */
public static List<LogicalRelationalOperator> getRelOpsFromPlan(
        LogicalPlan plan, Class<?> c) {
    Iterator<Operator> it = plan.getOperators();
    ArrayList<LogicalRelationalOperator> ops = 
        new ArrayList<LogicalRelationalOperator>();
    
    while(it.hasNext()){
        Operator op = it.next();
        if(op.getClass() == c){
            ops.add((LogicalRelationalOperator) op);
        }
        if(op instanceof LOForEach){
            ops.addAll(getRelOpsFromPlan(((LOForEach)op).getInnerPlan(), c));
        }
    }
    return ops;
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
public Map<LOForEach, Map<LogicalRelationalOperator, PhysicalOperator>> getForEachInnerLogToPhyMap(
        LogicalPlan plan) {
    Map<LOForEach, Map<LogicalRelationalOperator, PhysicalOperator>> result = Maps.newHashMap();
    Iterator<Operator> outerIter = plan.getOperators();
    while (outerIter.hasNext()) {
        Operator oper = outerIter.next();
        if (oper instanceof LOForEach) {
            LogicalPlan innerPlan = ((LOForEach) oper).getInnerPlan();
            Map<LogicalRelationalOperator, PhysicalOperator> innerOpMap = Maps.newHashMap();
            Iterator<Operator> innerIter = innerPlan.getOperators();
            while (innerIter.hasNext()) {
                Operator innerOper = innerIter.next();
                innerOpMap.put(((LogicalRelationalOperator) innerOper),
                        newLogToPhyMap.get(innerOper));
            }
            result.put((LOForEach) oper, innerOpMap);
        }
    }
    return result;
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
@Override
public boolean check(OperatorPlan matched) {
    LOLimit limit = (LOLimit) matched.getSources().get(0);
    // Match each foreach.
    List<Operator> preds = currentPlan.getPredecessors(limit);
    if (preds == null || preds.size() == 0)
        return false;

    Operator pred = preds.get(0);

    // Limit cannot be pushed up
    if (pred instanceof LOCogroup || pred instanceof LOFilter
            || pred instanceof LOSplit || pred instanceof LODistinct || pred instanceof LOJoin) {
        return false;
    }

    // Limit cannot be pushed in front of ForEach if it has a flatten
    if (pred instanceof LOForEach) {
        LOForEach foreach = (LOForEach) pred;
        LogicalPlan innerPlan = foreach.getInnerPlan();
        Iterator<Operator> it = innerPlan.getOperators();
        while (it.hasNext()) {
            Operator op = it.next();
            if (op instanceof LOGenerate) {
                LOGenerate gen = (LOGenerate) op;
                boolean[] flattenFlags = gen.getFlattenFlags();
                if (flattenFlags != null) {
                    for (boolean flatten : flattenFlags) {
                        if (flatten)
                            return false;
                    }
                }
            }
        }
    }
    return true;
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
@Test
public void testDistinct() throws Exception {
    //Test for bug where distinct wasn't being pruned properly causing union 
    //to fail to get a schema since the distinct relation had an incompatible schema
    //with the other relation being unioned.
    
    String testQuery = 
            "a = load 'd.txt' as (id, v1, v2);" +
            "b = load 'd.txt' as (id, v1, v2);" +
            "c = distinct a;" +
            "d = union b, c;" +
            "e = foreach d generate id, v1;" +
            "store e into 'empty';";
    
    //Generate optimized plan.
    LogicalPlan newLogicalPlan = buildPlan(testQuery);
    PlanOptimizer optimizer = new MyPlanOptimizer(newLogicalPlan, 3);
    optimizer.optimize();
    
    Iterator<Operator> iter = newLogicalPlan.getOperators();
    while (iter.hasNext()) {
        Operator o = iter.next();
        LogicalRelationalOperator lro = (LogicalRelationalOperator)o;
        if (lro == null || lro.getAlias() == null) continue;
        if (lro.getAlias().equals("d")) {
            assertNotNull(lro.getSchema());
        }
    }
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
@Override
public void visit(LOForEach foreach) throws FrontendException {
    appendOp( foreach ) ;
    
	boolean hasFlatten = false;
    LogicalPlan inner1 = foreach.getInnerPlan();
    Iterator<Operator> it = inner1.getOperators();
    while( it.hasNext() ) {
        Operator op = it.next();
        if( op instanceof LOGenerate ) {
            LOGenerate gen = (LOGenerate)op;
            boolean[] flattenFlags = gen.getFlattenFlags();
            if( flattenFlags != null ) {
                for( boolean flatten : flattenFlags ) {
                    if( flatten ) {
                        hasFlatten = true;
                        break;
                    }
                }
            }
        }
    }

	if (hasFlatten)
		sb.append(" [hasflat=\"true\"]");
	else
		sb.append(" [hasflat=\"false\"]");
    sb.append(";\n");
    appendEdges(foreach);
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
static void showLPOperators(LogicalPlan lp) {
   	System.out.println("Operators:");
   	Iterator<org.apache.pig.newplan.Operator> it = lp.getOperators();
   	while( it.hasNext() ) {
   		System.out.println("    op: " + it.next().getName());
   	}
   	System.out.println();
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
private int getForEachOperatorCount(LogicalPlan plan) {
    Iterator<Operator> ops = plan.getOperators();
    int count = 0;
    while( ops.hasNext() ) {
        Operator op = ops.next();
        if( op instanceof LOForEach )
            count++;
    }
    return count;
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
private LOForEach getForEachOperator(LogicalPlan plan) throws IOException {
    Iterator<Operator> ops = plan.getOperators();
    while( ops.hasNext() ) {
        Operator op = ops.next();
        if( op instanceof LOForEach ) {
            LOForEach foreach = (LOForEach)op;
            Operator succ = plan.getSuccessors( foreach ).get( 0 );
            if( !(succ instanceof LOForEach ) )
                return foreach;
        }
    }
    return null;
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
/**
 * returns first operator that is an instance of given class c
 * @param plan
 * @param c
 * @return instance of class c if any, or null
 * @throws IOException
 */
private Operator getOperator(LogicalPlan plan, Class<? extends Operator> c) throws IOException {
    Iterator<Operator> ops = plan.getOperators();
    while( ops.hasNext() ) {
        Operator op = ops.next();
        if( op.getClass().equals(c)) {
            return op;          
        }
    }
    return null;
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
private void testFull(String q, String partFilter, String filterExpr, boolean unsupportedExpr) throws Exception {
    TestLoader.partFilter = null;
    LogicalPlan newLogicalPlan = migrateAndOptimizePlan( q );

    if (partFilter != null) {
        Assert.assertEquals("checking partition filter:",
                partFilter,
                TestLoader.partFilter.toString());
    } else {
        Assert.assertTrue(TestLoader.partFilter == null);
    }

    if (filterExpr != null) {
        Operator op = newLogicalPlan.getSinks().get(0);
        LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);

        String actual = new PartitionFilterExtractor(null, new ArrayList<String>())
                .getExpression((LogicalExpression) filter.getFilterPlan().getSources().get(0)).toString();
        Assert.assertEquals("checking trimmed filter expression:",
                filterExpr, actual);
    } else {
        Iterator<Operator> it = newLogicalPlan.getOperators();
        while( it.hasNext() ) {
            Assert.assertFalse("Checking that filter has been removed since it contained" +
                    " only conditions on partition cols:",
                    (it.next() instanceof LOFilter));
        }
    }

    // Test that the filtered plan can be translated to physical plan (PIG-3657)
    LogToPhyTranslationVisitor translator = new LogToPhyTranslationVisitor(newLogicalPlan);
    translator.visit();
}