Java LogicalPlan.connect方法代码示例

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
public static void attachStorePlan(String scope, LogicalPlan lp, String fileName, String func, 
        Operator input, String alias, PigContext pigContext) throws FrontendException {
    func = func == null ? pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName()) : func;

    FuncSpec funcSpec = new FuncSpec( func );
    StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec( funcSpec );
    
    fileName = removeQuotes( fileName );
    FileSpec fileSpec = new FileSpec( fileName, funcSpec );
    String sig = alias + "_" + LogicalPlanBuilder.newOperatorKey(scope);
    stoFunc.setStoreFuncUDFContextSignature(sig);
    LOStore store = new LOStore(lp, fileSpec, stoFunc, sig);
    store.setAlias(alias);

    try {
        stoFunc.relToAbsPathForStoreLocation( fileName, getCurrentDir( pigContext ) );
    } catch (IOException ioe) {
        FrontendException e = new FrontendException(  ioe.getMessage(), ioe );
        throw e;
    }

    lp.add( store );
    lp.connect( input, store );
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
private LogicalPlan genNewLoadStorePlan(String inputFile,
                                        String outputFile, DataStorage dfs)
                                    throws Throwable {
    LogicalPlan plan = new LogicalPlan() ;
    FileSpec filespec1 =
        new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
    FileSpec filespec2 =
        new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
    LOLoad load = newLOLoad( filespec1, null, plan,
            ConfigurationUtil.toConfiguration(dfs.getConfiguration())) ;
    LOStore store = new LOStore(plan, filespec2, (StoreFuncInterface)PigContext.instantiateFuncFromSpec(filespec2.getFuncSpec()), null) ;

    plan.add(load) ;
    plan.add(store) ;

    plan.connect(load, store) ;

    return plan ;
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
public static void attachStorePlan(String scope, LogicalPlan lp, String fileName, String func,
        Operator input, String alias, PigContext pigContext) throws FrontendException {
    func = func == null ? pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName()) : func;

    FuncSpec funcSpec = new FuncSpec( func );
    StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec( funcSpec );

    fileName = removeQuotes( fileName );
    FileSpec fileSpec = new FileSpec( fileName, funcSpec );
    String sig = alias + "_" + LogicalPlanBuilder.newOperatorKey(scope);
    stoFunc.setStoreFuncUDFContextSignature(sig);
    LOStore store = new LOStore(lp, fileSpec, stoFunc, sig);
    store.setAlias(alias);

    try {
        stoFunc.relToAbsPathForStoreLocation( fileName, getCurrentDir( pigContext ) );
    } catch (IOException ioe) {
        FrontendException e = new FrontendException(  ioe.getMessage(), ioe );
        throw e;
    }

    lp.add( store );
    lp.connect( input, store );
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
public static LOForEach addForEachAfter(LogicalPlan plan, LogicalRelationalOperator op, int branch,
        Set<Integer> columnsToDrop) throws FrontendException {
    LOForEach foreach = new LOForEach(plan);
    
    plan.add(foreach);
    List<Operator> next = plan.getSuccessors(op);
    if (next != null) {
        LogicalRelationalOperator nextOp = (LogicalRelationalOperator)next.get(branch);
        plan.insertBetween(op, foreach, nextOp);
    }
    else {
        plan.connect(op, foreach);
    }
    
    LogicalPlan innerPlan = new LogicalPlan();
    foreach.setInnerPlan(innerPlan);
    
    LogicalSchema schema = op.getSchema();
    
    // build foreach inner plan
    List<LogicalExpressionPlan> exps = new ArrayList<LogicalExpressionPlan>();
    LOGenerate gen = new LOGenerate(innerPlan, exps, new boolean[schema.size()-columnsToDrop.size()]);
    innerPlan.add(gen);
    
    for (int i=0, j=0; i<schema.size(); i++) {
        if (columnsToDrop.contains(i)) {
            continue;
        }
        
        LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, i);
        innerPlan.add(innerLoad);
        innerPlan.connect(innerLoad, gen);
        
        LogicalExpressionPlan exp = new LogicalExpressionPlan();
        ProjectExpression prj = new ProjectExpression(exp, j++, -1, gen);
        exp.add(prj);
        exps.add(exp);
    }
    return foreach;
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
private void buildNestedOp(SourceLocation loc, LogicalPlan plan, LogicalRelationalOperator op,
        String alias, Operator inputOp) {
    op.setLocation( loc );
    setAlias( op, alias );
    plan.add( op );
    plan.connect( inputOp, op );
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
@Override
protected OperatorPlan buildPattern() {
    // the pattern that this rule looks for
    // is foreach -> filter
    LogicalPlan plan = new LogicalPlan();
    LogicalRelationalOperator foreach = new LOForEach(plan);
    LogicalRelationalOperator filter = new LOFilter(plan);

    plan.add(foreach);
    plan.add(filter);
    plan.connect(foreach, filter);

    return plan;
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
@Test
public void testDistinct1() throws Throwable {
    printCurrentMethodName();
    LogicalPlan plan = new LogicalPlan();

    String pigStorage = PigStorage.class.getName();

    LOLoad load1 = newLOLoad(
            new FileSpec("pi", new FuncSpec(pigStorage)),
            null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
    );

    // schema for input#1
    Schema inputSchema1 = null;
    {
        List<FieldSchema> innerList = new ArrayList<FieldSchema>();
        innerList.add(new FieldSchema("innerfield1", DataType.BAG));
        innerList.add(new FieldSchema("innerfield2", DataType.FLOAT));
        Schema innerSchema = new Schema(innerList);

        List<FieldSchema> fsList1 = new ArrayList<FieldSchema>();
        fsList1.add(new FieldSchema("field1", DataType.INTEGER));
        fsList1.add(new FieldSchema("field2", DataType.BYTEARRAY));
        fsList1.add(new FieldSchema("field3", innerSchema));
        fsList1.add(new FieldSchema("field4", DataType.BAG));

        inputSchema1 = new Schema(fsList1);
    }

    // set schemas
    load1.setSchema(org.apache.pig.newplan.logical.Util.translateSchema(inputSchema1));

    // create union operator
    LODistinct distinct1 = new LODistinct(plan);

    // wiring
    plan.add(load1);
    plan.add(distinct1);

    plan.connect(load1, distinct1);

    // validate
    CompilationMessageCollector collector = new CompilationMessageCollector();
    TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
    typeChecker.visit();
    printMessageCollector(collector);

    // check end result schema
    LogicalSchema outputSchema = distinct1.getSchema();
    assertTrue(load1.getSchema().isEqual(outputSchema));
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
/**
 * Build a plan for the given alias. Extra branches and child branch under alias
 * will be ignored. Dependent branch (i.e. scalar) will be kept.
 * @throws IOException
 */
void buildPlan(String alias) throws IOException {
    if( alias == null )
        skipStores();

    final Queue<Operator> queue = new LinkedList<Operator>();
    if( alias != null ) {
        Operator op = getOperator( alias );
        if (op == null) {
            String msg = "Unable to find an operator for alias " + alias;
            throw new FrontendException( msg, 1003, PigException.INPUT );
        }
        queue.add( op );
    } else {
        List<Operator> sinks = lp.getSinks();
        if( sinks != null ) {
            for( Operator sink : sinks ) {
                if( sink instanceof LOStore )
                    queue.add( sink );
            }
        }
    }

    LogicalPlan plan = new LogicalPlan();

    while( !queue.isEmpty() ) {
        Operator currOp = queue.poll();
        plan.add( currOp );

        List<Operator> preds = lp.getPredecessors( currOp );
        if( preds != null ) {
            List<Operator> ops = new ArrayList<Operator>( preds );
            for( Operator pred : ops ) {
                if( !queue.contains( pred ) )
                    queue.add( pred );
                plan.connect( pred, currOp );
            }
        }

        // visit expression associated with currOp. If it refers to any other operator
        // that operator is also going to be enqueued.
        currOp.accept( new AllExpressionVisitor( plan, new DependencyOrderWalker( plan ) ) {
                @Override
                protected LogicalExpressionVisitor getVisitor(LogicalExpressionPlan exprPlan)
                throws FrontendException {
                    return new LogicalExpressionVisitor( exprPlan, new DependencyOrderWalker( exprPlan ) ) {
                        @Override
                        public void visit(ScalarExpression expr) throws FrontendException {
                            Operator refOp = expr.getImplicitReferencedOperator();
                            if( !queue.contains( refOp ) )
                                queue.add( refOp );
                        }
                    };
                }
            }
        );

        currOp.setPlan( plan );
    }
    lp = plan;
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
public void testFilterWithInnerPlan(byte field1Type, byte field2Type) throws Throwable {

            // Create outer plan
            printCurrentMethodName();
            LogicalPlan plan = new LogicalPlan();

            String pigStorage = PigStorage.class.getName();

            LOLoad load1 = newLOLoad(
                    new FileSpec("pi", new FuncSpec(pigStorage)),
                    null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
            );
            // schema for input#1
            Schema inputSchema1 = null;
            {
                List<FieldSchema> fsList1 = new ArrayList<FieldSchema>();
                fsList1.add(new FieldSchema("field1", field1Type));
                fsList1.add(new FieldSchema("field2", field2Type));

                inputSchema1 = new Schema(fsList1);
            }

            // set schemas
            load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema(inputSchema1));

            // Create inner plan
            LogicalExpressionPlan innerPlan = new LogicalExpressionPlan();
            // filter
            LOFilter filter1 = new LOFilter(plan);
            filter1.setFilterPlan(innerPlan);

            ProjectExpression project1 = new ProjectExpression(innerPlan, 0, 0, filter1);
            ProjectExpression project2 = new ProjectExpression(innerPlan, 0, 1, filter1);

            GreaterThanExpression gt1 = new GreaterThanExpression(innerPlan, project1, project2);

            plan.add(load1);
            plan.add(filter1);
            plan.connect(load1, filter1);

            CompilationMessageCollector collector = new CompilationMessageCollector();
            TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
            typeChecker.visit();
            printMessageCollector(collector);
            //printTypeGraph(plan);

            if (collector.hasError()) {
                throw new AssertionError("Expect no error");
            }

            LogicalSchema endResultSchema = filter1.getSchema();
            assertEquals(endResultSchema.getField(0).type, field1Type);
            assertEquals(endResultSchema.getField(1).type, field2Type);

        }
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
@Test
public void testJoinDifferentInner() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();
    LogicalSchema jaschema1 = new LogicalSchema();
    jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema1, lp, conf);
    lp.add(A1);

    // B = load
    LogicalSchema jbschema1 = new LogicalSchema();
    jbschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LOLoad B1 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema1, lp, conf);
    lp.add(B1);

    // C = join
    LogicalSchema jcschema1 = new LogicalSchema();
    jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan1 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan1, 0, 0, null);
    LogicalExpressionPlan bprojplan1 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan1, 1, 0, null);
    MultiMap<Integer, LogicalExpressionPlan> mm1 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm1.put(0, aprojplan1);
    mm1.put(1, bprojplan1);
    LOJoin C1 = new LOJoin(lp, mm1, JOINTYPE.HASH, new boolean[] { true, true });
    C1.neverUseForRealSetSchema(jcschema1);
    lp.add(C1);
    lp.connect(A1, C1);
    lp.connect(B1, C1);

    // Test different inner status
    // A = load
    LogicalSchema jaschema3 = new LogicalSchema();
    jaschema3.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A3 = newLOLoad(new FileSpec("/abc",
            new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
                            "x", "y" })), jaschema3, lp, conf);
    lp.add(A3);

    // B = load
    LogicalSchema jbschema3 = new LogicalSchema();
    jbschema3.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LOLoad B3 = newLOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema3, lp, conf);
    lp.add(B3);

    // C = join
    LogicalSchema jcschema3 = new LogicalSchema();
    jcschema3.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    jcschema3.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
    LogicalExpressionPlan aprojplan3 = new LogicalExpressionPlan();
    new ProjectExpression(aprojplan3, 0, 0, null);
    LogicalExpressionPlan bprojplan3 = new LogicalExpressionPlan();
    new ProjectExpression(bprojplan3, 1, 0, null);
    MultiMap<Integer, LogicalExpressionPlan> mm3 =
            new MultiMap<Integer, LogicalExpressionPlan>();
    mm3.put(0, aprojplan3);
    mm3.put(1, bprojplan3);
    LOJoin C3 = new LOJoin(lp, mm3, JOINTYPE.HASH, new boolean[] { true, false });
    C3.neverUseForRealSetSchema(jcschema3);
    lp.add(C3);
    lp.connect(A3, C3);
    lp.connect(B3, C3);

    assertFalse(C1.isEqual(C3));
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
@Test
public void testRelationalSameOpDifferentPreds() throws FrontendException {
    LogicalPlan lp1 = new LogicalPlan();
    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad A1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp1,
            conf);
    lp1.add(A1);

    LogicalExpressionPlan fp1 = new LogicalExpressionPlan();
    ProjectExpression fy1 = new ProjectExpression(fp1, 0, 0, null);
    ConstantExpression fc1 = new ConstantExpression(fp1,
            new Integer(0));
    new EqualExpression(fp1, fy1, fc1);
    LOFilter D1 = new LOFilter(lp1, fp1);
    LogicalSchema cschema = new LogicalSchema();
    cschema.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    D1.neverUseForRealSetSchema(cschema);
    lp1.add(D1);
    lp1.connect(A1, D1);

    LogicalPlan lp2 = new LogicalPlan();
    LOLoad A2 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), null, lp2,
            conf);
    lp2.add(A2);

    LogicalExpressionPlan fp2 = new LogicalExpressionPlan();
    ProjectExpression fy2 = new ProjectExpression(fp2, 0, 0, null);
    ConstantExpression fc2 = new ConstantExpression(fp2,
            new Integer(0));
    new EqualExpression(fp2, fy2, fc2);
    LOFilter D2 = new LOFilter(lp2, fp2);
    D2.neverUseForRealSetSchema(cschema);
    lp2.add(D2);
    lp2.connect(A2, D2);

    assertTrue(D1.isEqual(D2));
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
@Test
public void testFilterWithInnerPlan3() throws Throwable {

    // Create outer plan
    printCurrentMethodName();
    LogicalPlan plan = new LogicalPlan();

    String pigStorage = PigStorage.class.getName();

    LOLoad load1 = newLOLoad(
            new FileSpec("pi", new FuncSpec(pigStorage)),
            null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
    );

    // schema for input#1
    Schema inputSchema1 = null;
    {
        List<FieldSchema> fsList1 = new ArrayList<FieldSchema>();
        fsList1.add(new FieldSchema("field1", DataType.INTEGER));
        fsList1.add(new FieldSchema("field2", DataType.LONG));

        inputSchema1 = new Schema(fsList1);
    }

    // set schemas
    load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1)));

    // Create inner plan
    LogicalExpressionPlan innerPlan = new LogicalExpressionPlan();

    // filter
    LOFilter filter1 = new LOFilter(plan);
    filter1.setFilterPlan(innerPlan);

    ProjectExpression project1 = new ProjectExpression(innerPlan, 0, 0, filter1);
    ProjectExpression project2 = new ProjectExpression(innerPlan, 0, 1, filter1);

    AddExpression add1 = new AddExpression(innerPlan, project1, project2);

    plan.add(load1);
    plan.add(filter1);
    plan.connect(load1, filter1);

    CompilationMessageCollector collector = new CompilationMessageCollector();
    TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
    try {
        typeChecker.visit();
    } catch (Exception t) {
        // good
    }
    printMessageCollector(collector);
    //printTypeGraph(plan);

    if (!collector.hasError()) {
        throw new AssertionError("Expect error");
    }

}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
@Test
public void testSortWithInnerPlan3() throws Throwable {

    // Create outer plan
    printCurrentMethodName();
    LogicalPlan plan = new LogicalPlan();

    String pigStorage = PigStorage.class.getName();

    LOLoad load1 =  newLOLoad(
            new FileSpec("pi", new FuncSpec(pigStorage)),
            null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
    );


    // schema for input#1
    Schema inputSchema1 = null;
    {
        List<FieldSchema> fsList1 = new ArrayList<FieldSchema>();
        fsList1.add(new FieldSchema("field1", DataType.BYTEARRAY));
        fsList1.add(new FieldSchema("field2", DataType.INTEGER));

        inputSchema1 = new Schema(fsList1);
    }

    // set schemas
    load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1)));;


    // Sort
    LOSort sort1 = new LOSort(plan);

    // Create expression inner plan #1
    LogicalExpressionPlan innerPlan1 = new LogicalExpressionPlan();
    ProjectExpression project11 = new ProjectExpression(innerPlan1, 0, 0, sort1);
    ProjectExpression project12 = new ProjectExpression(innerPlan1, 0, 1, sort1);
    MultiplyExpression mul1 = new MultiplyExpression(innerPlan1, project11, project12);


    // Create expression inner plan #2
    LogicalExpressionPlan innerPlan2 = new LogicalExpressionPlan();
    ProjectExpression project21 = new ProjectExpression(innerPlan2, 0, 0, sort1);
    ConstantExpression const21 = new ConstantExpression(innerPlan2, "26");
    ModExpression mod21 = new ModExpression(innerPlan2, project21, const21);

    // List of innerplans
    List<LogicalExpressionPlan> innerPlans = new ArrayList<LogicalExpressionPlan>();
    innerPlans.add(innerPlan1);
    innerPlans.add(innerPlan2);

    // List of ASC flags
    List<Boolean> ascList = new ArrayList<Boolean>();
    ascList.add(true);
    ascList.add(true);

    // Sort
    sort1.setAscendingCols(ascList);
    sort1.setSortColPlans(innerPlans);

    plan.add(load1);
    plan.add(sort1);
    plan.connect(load1, sort1);

    CompilationMessageCollector collector = new CompilationMessageCollector();
    TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
    try {
        typeChecker.visit();
        fail("Error expected");
    } catch (Exception t) {
        // good
    }
    printMessageCollector(collector);
    //printTypeGraph(plan);

    if (!collector.hasError()) {
        throw new AssertionError("Error expected");
    }
}
 

import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
public static LOForEach addForEachAfter(LogicalPlan plan, LogicalRelationalOperator op, int branch,
        Set<Integer> columnsToDrop) throws FrontendException {
    LOForEach foreach = new LOForEach(plan);

    plan.add(foreach);
    List<Operator> next = plan.getSuccessors(op);
    if (next != null) {
        LogicalRelationalOperator nextOp = (LogicalRelationalOperator)next.get(branch);
        plan.insertBetween(op, foreach, nextOp);
        foreach.setAlias(op.getAlias());
    }
    else {
        plan.connect(op, foreach);
    }

    LogicalPlan innerPlan = new LogicalPlan();
    foreach.setInnerPlan(innerPlan);

    LogicalSchema schema = op.getSchema();

    // build foreach inner plan
    List<LogicalExpressionPlan> exps = new ArrayList<LogicalExpressionPlan>();
    LOGenerate gen = new LOGenerate(innerPlan, exps, new boolean[schema.size()-columnsToDrop.size()]);
    innerPlan.add(gen);

    for (int i=0, j=0; i<schema.size(); i++) {
        if (columnsToDrop.contains(i)) {
            continue;
        }

        LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, i);
        innerPlan.add(innerLoad);
        innerPlan.connect(innerLoad, gen);

        LogicalExpressionPlan exp = new LogicalExpressionPlan();
        ProjectExpression prj = new ProjectExpression(exp, j++, -1, gen);
        exp.add(prj);
        exps.add(exp);
    }
    return foreach;
}