本文整理汇总了Java中org.apache.pig.newplan.logical.relational.LogicalPlan.connect方法的典型用法代码示例。如果您正苦于以下问题:Java LogicalPlan.connect方法的具体用法?Java LogicalPlan.connect怎么用?Java LogicalPlan.connect使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.pig.newplan.logical.relational.LogicalPlan
的用法示例。
在下文中一共展示了LogicalPlan.connect方法的14个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: attachStorePlan
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
public static void attachStorePlan(String scope, LogicalPlan lp, String fileName, String func,
Operator input, String alias, PigContext pigContext) throws FrontendException {
func = func == null ? pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName()) : func;
FuncSpec funcSpec = new FuncSpec( func );
StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec( funcSpec );
fileName = removeQuotes( fileName );
FileSpec fileSpec = new FileSpec( fileName, funcSpec );
String sig = alias + "_" + LogicalPlanBuilder.newOperatorKey(scope);
stoFunc.setStoreFuncUDFContextSignature(sig);
LOStore store = new LOStore(lp, fileSpec, stoFunc, sig);
store.setAlias(alias);
try {
stoFunc.relToAbsPathForStoreLocation( fileName, getCurrentDir( pigContext ) );
} catch (IOException ioe) {
FrontendException e = new FrontendException( ioe.getMessage(), ioe );
throw e;
}
lp.add( store );
lp.connect( input, store );
}
示例2: genNewLoadStorePlan
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
private LogicalPlan genNewLoadStorePlan(String inputFile,
String outputFile, DataStorage dfs)
throws Throwable {
LogicalPlan plan = new LogicalPlan() ;
FileSpec filespec1 =
new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
FileSpec filespec2 =
new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
LOLoad load = newLOLoad( filespec1, null, plan,
ConfigurationUtil.toConfiguration(dfs.getConfiguration())) ;
LOStore store = new LOStore(plan, filespec2, (StoreFuncInterface)PigContext.instantiateFuncFromSpec(filespec2.getFuncSpec()), null) ;
plan.add(load) ;
plan.add(store) ;
plan.connect(load, store) ;
return plan ;
}
示例3: attachStorePlan
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
public static void attachStorePlan(String scope, LogicalPlan lp, String fileName, String func,
Operator input, String alias, PigContext pigContext) throws FrontendException {
func = func == null ? pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName()) : func;
FuncSpec funcSpec = new FuncSpec( func );
StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec( funcSpec );
fileName = removeQuotes( fileName );
FileSpec fileSpec = new FileSpec( fileName, funcSpec );
String sig = alias + "_" + LogicalPlanBuilder.newOperatorKey(scope);
stoFunc.setStoreFuncUDFContextSignature(sig);
LOStore store = new LOStore(lp, fileSpec, stoFunc, sig);
store.setAlias(alias);
try {
stoFunc.relToAbsPathForStoreLocation( fileName, getCurrentDir( pigContext ) );
} catch (IOException ioe) {
FrontendException e = new FrontendException( ioe.getMessage(), ioe );
throw e;
}
lp.add( store );
lp.connect( input, store );
}
示例4: addForEachAfter
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
public static LOForEach addForEachAfter(LogicalPlan plan, LogicalRelationalOperator op, int branch,
Set<Integer> columnsToDrop) throws FrontendException {
LOForEach foreach = new LOForEach(plan);
plan.add(foreach);
List<Operator> next = plan.getSuccessors(op);
if (next != null) {
LogicalRelationalOperator nextOp = (LogicalRelationalOperator)next.get(branch);
plan.insertBetween(op, foreach, nextOp);
}
else {
plan.connect(op, foreach);
}
LogicalPlan innerPlan = new LogicalPlan();
foreach.setInnerPlan(innerPlan);
LogicalSchema schema = op.getSchema();
// build foreach inner plan
List<LogicalExpressionPlan> exps = new ArrayList<LogicalExpressionPlan>();
LOGenerate gen = new LOGenerate(innerPlan, exps, new boolean[schema.size()-columnsToDrop.size()]);
innerPlan.add(gen);
for (int i=0, j=0; i<schema.size(); i++) {
if (columnsToDrop.contains(i)) {
continue;
}
LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, i);
innerPlan.add(innerLoad);
innerPlan.connect(innerLoad, gen);
LogicalExpressionPlan exp = new LogicalExpressionPlan();
ProjectExpression prj = new ProjectExpression(exp, j++, -1, gen);
exp.add(prj);
exps.add(exp);
}
return foreach;
}
示例5: buildNestedOp
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
private void buildNestedOp(SourceLocation loc, LogicalPlan plan, LogicalRelationalOperator op,
String alias, Operator inputOp) {
op.setLocation( loc );
setAlias( op, alias );
plan.add( op );
plan.connect( inputOp, op );
}
示例6: buildPattern
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
@Override
protected OperatorPlan buildPattern() {
// the pattern that this rule looks for
// is foreach -> filter
LogicalPlan plan = new LogicalPlan();
LogicalRelationalOperator foreach = new LOForEach(plan);
LogicalRelationalOperator filter = new LOFilter(plan);
plan.add(foreach);
plan.add(filter);
plan.connect(foreach, filter);
return plan;
}
示例7: testDistinct1
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
@Test
public void testDistinct1() throws Throwable {
printCurrentMethodName();
LogicalPlan plan = new LogicalPlan();
String pigStorage = PigStorage.class.getName();
LOLoad load1 = newLOLoad(
new FileSpec("pi", new FuncSpec(pigStorage)),
null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
);
// schema for input#1
Schema inputSchema1 = null;
{
List<FieldSchema> innerList = new ArrayList<FieldSchema>();
innerList.add(new FieldSchema("innerfield1", DataType.BAG));
innerList.add(new FieldSchema("innerfield2", DataType.FLOAT));
Schema innerSchema = new Schema(innerList);
List<FieldSchema> fsList1 = new ArrayList<FieldSchema>();
fsList1.add(new FieldSchema("field1", DataType.INTEGER));
fsList1.add(new FieldSchema("field2", DataType.BYTEARRAY));
fsList1.add(new FieldSchema("field3", innerSchema));
fsList1.add(new FieldSchema("field4", DataType.BAG));
inputSchema1 = new Schema(fsList1);
}
// set schemas
load1.setSchema(org.apache.pig.newplan.logical.Util.translateSchema(inputSchema1));
// create union operator
LODistinct distinct1 = new LODistinct(plan);
// wiring
plan.add(load1);
plan.add(distinct1);
plan.connect(load1, distinct1);
// validate
CompilationMessageCollector collector = new CompilationMessageCollector();
TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
typeChecker.visit();
printMessageCollector(collector);
// check end result schema
LogicalSchema outputSchema = distinct1.getSchema();
assertTrue(load1.getSchema().isEqual(outputSchema));
}
示例8: buildPlan
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
/**
* Build a plan for the given alias. Extra branches and child branch under alias
* will be ignored. Dependent branch (i.e. scalar) will be kept.
* @throws IOException
*/
void buildPlan(String alias) throws IOException {
if( alias == null )
skipStores();
final Queue<Operator> queue = new LinkedList<Operator>();
if( alias != null ) {
Operator op = getOperator( alias );
if (op == null) {
String msg = "Unable to find an operator for alias " + alias;
throw new FrontendException( msg, 1003, PigException.INPUT );
}
queue.add( op );
} else {
List<Operator> sinks = lp.getSinks();
if( sinks != null ) {
for( Operator sink : sinks ) {
if( sink instanceof LOStore )
queue.add( sink );
}
}
}
LogicalPlan plan = new LogicalPlan();
while( !queue.isEmpty() ) {
Operator currOp = queue.poll();
plan.add( currOp );
List<Operator> preds = lp.getPredecessors( currOp );
if( preds != null ) {
List<Operator> ops = new ArrayList<Operator>( preds );
for( Operator pred : ops ) {
if( !queue.contains( pred ) )
queue.add( pred );
plan.connect( pred, currOp );
}
}
// visit expression associated with currOp. If it refers to any other operator
// that operator is also going to be enqueued.
currOp.accept( new AllExpressionVisitor( plan, new DependencyOrderWalker( plan ) ) {
@Override
protected LogicalExpressionVisitor getVisitor(LogicalExpressionPlan exprPlan)
throws FrontendException {
return new LogicalExpressionVisitor( exprPlan, new DependencyOrderWalker( exprPlan ) ) {
@Override
public void visit(ScalarExpression expr) throws FrontendException {
Operator refOp = expr.getImplicitReferencedOperator();
if( !queue.contains( refOp ) )
queue.add( refOp );
}
};
}
}
);
currOp.setPlan( plan );
}
lp = plan;
}
示例9: testFilterWithInnerPlan
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
public void testFilterWithInnerPlan(byte field1Type, byte field2Type) throws Throwable {
// Create outer plan
printCurrentMethodName();
LogicalPlan plan = new LogicalPlan();
String pigStorage = PigStorage.class.getName();
LOLoad load1 = newLOLoad(
new FileSpec("pi", new FuncSpec(pigStorage)),
null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
);
// schema for input#1
Schema inputSchema1 = null;
{
List<FieldSchema> fsList1 = new ArrayList<FieldSchema>();
fsList1.add(new FieldSchema("field1", field1Type));
fsList1.add(new FieldSchema("field2", field2Type));
inputSchema1 = new Schema(fsList1);
}
// set schemas
load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema(inputSchema1));
// Create inner plan
LogicalExpressionPlan innerPlan = new LogicalExpressionPlan();
// filter
LOFilter filter1 = new LOFilter(plan);
filter1.setFilterPlan(innerPlan);
ProjectExpression project1 = new ProjectExpression(innerPlan, 0, 0, filter1);
ProjectExpression project2 = new ProjectExpression(innerPlan, 0, 1, filter1);
GreaterThanExpression gt1 = new GreaterThanExpression(innerPlan, project1, project2);
plan.add(load1);
plan.add(filter1);
plan.connect(load1, filter1);
CompilationMessageCollector collector = new CompilationMessageCollector();
TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
typeChecker.visit();
printMessageCollector(collector);
//printTypeGraph(plan);
if (collector.hasError()) {
throw new AssertionError("Expect no error");
}
LogicalSchema endResultSchema = filter1.getSchema();
assertEquals(endResultSchema.getField(0).type, field1Type);
assertEquals(endResultSchema.getField(1).type, field2Type);
}
示例10: testJoinDifferentInner
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
@Test
public void testJoinDifferentInner() throws FrontendException {
LogicalPlan lp = new LogicalPlan();
LogicalSchema jaschema1 = new LogicalSchema();
jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
"x", null, DataType.INTEGER));
LOLoad A1 = newLOLoad(new FileSpec("/abc",
new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
"x", "y" })), jaschema1, lp, conf);
lp.add(A1);
// B = load
LogicalSchema jbschema1 = new LogicalSchema();
jbschema1.addField(new LogicalSchema.LogicalFieldSchema(
"y", null, DataType.INTEGER));
LOLoad B1 = newLOLoad(new FileSpec("/def",
new FuncSpec("PigStorage", "\t")), jbschema1, lp, conf);
lp.add(B1);
// C = join
LogicalSchema jcschema1 = new LogicalSchema();
jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
"x", null, DataType.INTEGER));
jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
"y", null, DataType.INTEGER));
LogicalExpressionPlan aprojplan1 = new LogicalExpressionPlan();
new ProjectExpression(aprojplan1, 0, 0, null);
LogicalExpressionPlan bprojplan1 = new LogicalExpressionPlan();
new ProjectExpression(bprojplan1, 1, 0, null);
MultiMap<Integer, LogicalExpressionPlan> mm1 =
new MultiMap<Integer, LogicalExpressionPlan>();
mm1.put(0, aprojplan1);
mm1.put(1, bprojplan1);
LOJoin C1 = new LOJoin(lp, mm1, JOINTYPE.HASH, new boolean[] { true, true });
C1.neverUseForRealSetSchema(jcschema1);
lp.add(C1);
lp.connect(A1, C1);
lp.connect(B1, C1);
// Test different inner status
// A = load
LogicalSchema jaschema3 = new LogicalSchema();
jaschema3.addField(new LogicalSchema.LogicalFieldSchema(
"x", null, DataType.INTEGER));
LOLoad A3 = newLOLoad(new FileSpec("/abc",
new FuncSpec("org.apache.pig.test.TestNewPlanOperatorPlan$FooLoad", new String[] {
"x", "y" })), jaschema3, lp, conf);
lp.add(A3);
// B = load
LogicalSchema jbschema3 = new LogicalSchema();
jbschema3.addField(new LogicalSchema.LogicalFieldSchema(
"y", null, DataType.INTEGER));
LOLoad B3 = newLOLoad(new FileSpec("/def",
new FuncSpec("PigStorage", "\t")), jbschema3, lp, conf);
lp.add(B3);
// C = join
LogicalSchema jcschema3 = new LogicalSchema();
jcschema3.addField(new LogicalSchema.LogicalFieldSchema(
"x", null, DataType.INTEGER));
jcschema3.addField(new LogicalSchema.LogicalFieldSchema(
"y", null, DataType.INTEGER));
LogicalExpressionPlan aprojplan3 = new LogicalExpressionPlan();
new ProjectExpression(aprojplan3, 0, 0, null);
LogicalExpressionPlan bprojplan3 = new LogicalExpressionPlan();
new ProjectExpression(bprojplan3, 1, 0, null);
MultiMap<Integer, LogicalExpressionPlan> mm3 =
new MultiMap<Integer, LogicalExpressionPlan>();
mm3.put(0, aprojplan3);
mm3.put(1, bprojplan3);
LOJoin C3 = new LOJoin(lp, mm3, JOINTYPE.HASH, new boolean[] { true, false });
C3.neverUseForRealSetSchema(jcschema3);
lp.add(C3);
lp.connect(A3, C3);
lp.connect(B3, C3);
assertFalse(C1.isEqual(C3));
}
示例11: testRelationalSameOpDifferentPreds
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
@Test
public void testRelationalSameOpDifferentPreds() throws FrontendException {
LogicalPlan lp1 = new LogicalPlan();
LogicalSchema aschema1 = new LogicalSchema();
aschema1.addField(new LogicalSchema.LogicalFieldSchema(
"x", null, DataType.INTEGER));
LOLoad A1 = newLOLoad(new FileSpec("/abc",
new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp1,
conf);
lp1.add(A1);
LogicalExpressionPlan fp1 = new LogicalExpressionPlan();
ProjectExpression fy1 = new ProjectExpression(fp1, 0, 0, null);
ConstantExpression fc1 = new ConstantExpression(fp1,
new Integer(0));
new EqualExpression(fp1, fy1, fc1);
LOFilter D1 = new LOFilter(lp1, fp1);
LogicalSchema cschema = new LogicalSchema();
cschema.addField(new LogicalSchema.LogicalFieldSchema(
"x", null, DataType.INTEGER));
D1.neverUseForRealSetSchema(cschema);
lp1.add(D1);
lp1.connect(A1, D1);
LogicalPlan lp2 = new LogicalPlan();
LOLoad A2 = newLOLoad(new FileSpec("/abc",
new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), null, lp2,
conf);
lp2.add(A2);
LogicalExpressionPlan fp2 = new LogicalExpressionPlan();
ProjectExpression fy2 = new ProjectExpression(fp2, 0, 0, null);
ConstantExpression fc2 = new ConstantExpression(fp2,
new Integer(0));
new EqualExpression(fp2, fy2, fc2);
LOFilter D2 = new LOFilter(lp2, fp2);
D2.neverUseForRealSetSchema(cschema);
lp2.add(D2);
lp2.connect(A2, D2);
assertTrue(D1.isEqual(D2));
}
示例12: testFilterWithInnerPlan3
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
@Test
public void testFilterWithInnerPlan3() throws Throwable {
// Create outer plan
printCurrentMethodName();
LogicalPlan plan = new LogicalPlan();
String pigStorage = PigStorage.class.getName();
LOLoad load1 = newLOLoad(
new FileSpec("pi", new FuncSpec(pigStorage)),
null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
);
// schema for input#1
Schema inputSchema1 = null;
{
List<FieldSchema> fsList1 = new ArrayList<FieldSchema>();
fsList1.add(new FieldSchema("field1", DataType.INTEGER));
fsList1.add(new FieldSchema("field2", DataType.LONG));
inputSchema1 = new Schema(fsList1);
}
// set schemas
load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1)));
// Create inner plan
LogicalExpressionPlan innerPlan = new LogicalExpressionPlan();
// filter
LOFilter filter1 = new LOFilter(plan);
filter1.setFilterPlan(innerPlan);
ProjectExpression project1 = new ProjectExpression(innerPlan, 0, 0, filter1);
ProjectExpression project2 = new ProjectExpression(innerPlan, 0, 1, filter1);
AddExpression add1 = new AddExpression(innerPlan, project1, project2);
plan.add(load1);
plan.add(filter1);
plan.connect(load1, filter1);
CompilationMessageCollector collector = new CompilationMessageCollector();
TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
try {
typeChecker.visit();
} catch (Exception t) {
// good
}
printMessageCollector(collector);
//printTypeGraph(plan);
if (!collector.hasError()) {
throw new AssertionError("Expect error");
}
}
示例13: testSortWithInnerPlan3
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
@Test
public void testSortWithInnerPlan3() throws Throwable {
// Create outer plan
printCurrentMethodName();
LogicalPlan plan = new LogicalPlan();
String pigStorage = PigStorage.class.getName();
LOLoad load1 = newLOLoad(
new FileSpec("pi", new FuncSpec(pigStorage)),
null, plan, new Configuration(ConfigurationUtil.toConfiguration(pc.getFs().getConfiguration()))
);
// schema for input#1
Schema inputSchema1 = null;
{
List<FieldSchema> fsList1 = new ArrayList<FieldSchema>();
fsList1.add(new FieldSchema("field1", DataType.BYTEARRAY));
fsList1.add(new FieldSchema("field2", DataType.INTEGER));
inputSchema1 = new Schema(fsList1);
}
// set schemas
load1.setScriptSchema(org.apache.pig.newplan.logical.Util.translateSchema((inputSchema1)));;
// Sort
LOSort sort1 = new LOSort(plan);
// Create expression inner plan #1
LogicalExpressionPlan innerPlan1 = new LogicalExpressionPlan();
ProjectExpression project11 = new ProjectExpression(innerPlan1, 0, 0, sort1);
ProjectExpression project12 = new ProjectExpression(innerPlan1, 0, 1, sort1);
MultiplyExpression mul1 = new MultiplyExpression(innerPlan1, project11, project12);
// Create expression inner plan #2
LogicalExpressionPlan innerPlan2 = new LogicalExpressionPlan();
ProjectExpression project21 = new ProjectExpression(innerPlan2, 0, 0, sort1);
ConstantExpression const21 = new ConstantExpression(innerPlan2, "26");
ModExpression mod21 = new ModExpression(innerPlan2, project21, const21);
// List of innerplans
List<LogicalExpressionPlan> innerPlans = new ArrayList<LogicalExpressionPlan>();
innerPlans.add(innerPlan1);
innerPlans.add(innerPlan2);
// List of ASC flags
List<Boolean> ascList = new ArrayList<Boolean>();
ascList.add(true);
ascList.add(true);
// Sort
sort1.setAscendingCols(ascList);
sort1.setSortColPlans(innerPlans);
plan.add(load1);
plan.add(sort1);
plan.connect(load1, sort1);
CompilationMessageCollector collector = new CompilationMessageCollector();
TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
try {
typeChecker.visit();
fail("Error expected");
} catch (Exception t) {
// good
}
printMessageCollector(collector);
//printTypeGraph(plan);
if (!collector.hasError()) {
throw new AssertionError("Error expected");
}
}
示例14: addForEachAfter
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入方法依赖的package包/类
public static LOForEach addForEachAfter(LogicalPlan plan, LogicalRelationalOperator op, int branch,
Set<Integer> columnsToDrop) throws FrontendException {
LOForEach foreach = new LOForEach(plan);
plan.add(foreach);
List<Operator> next = plan.getSuccessors(op);
if (next != null) {
LogicalRelationalOperator nextOp = (LogicalRelationalOperator)next.get(branch);
plan.insertBetween(op, foreach, nextOp);
foreach.setAlias(op.getAlias());
}
else {
plan.connect(op, foreach);
}
LogicalPlan innerPlan = new LogicalPlan();
foreach.setInnerPlan(innerPlan);
LogicalSchema schema = op.getSchema();
// build foreach inner plan
List<LogicalExpressionPlan> exps = new ArrayList<LogicalExpressionPlan>();
LOGenerate gen = new LOGenerate(innerPlan, exps, new boolean[schema.size()-columnsToDrop.size()]);
innerPlan.add(gen);
for (int i=0, j=0; i<schema.size(); i++) {
if (columnsToDrop.contains(i)) {
continue;
}
LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, i);
innerPlan.add(innerLoad);
innerPlan.connect(innerLoad, gen);
LogicalExpressionPlan exp = new LogicalExpressionPlan();
ProjectExpression prj = new ProjectExpression(exp, j++, -1, gen);
exp.add(prj);
exps.add(exp);
}
return foreach;
}