本文整理汇总了Java中org.apache.pig.newplan.logical.relational.LogicalPlan类的典型用法代码示例。如果您正苦于以下问题:Java LogicalPlan类的具体用法?Java LogicalPlan怎么用?Java LogicalPlan使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
LogicalPlan类属于org.apache.pig.newplan.logical.relational包,在下文中一共展示了LogicalPlan类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: testSimpleNotPossible
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testSimpleNotPossible() throws Exception {
String query = "A =LOAD 'file.txt' AS (name, cuisines:bag{ t : ( cuisine ) } );" +
"B = FOREACH A GENERATE name, flatten(cuisines) as cuisines;" +
"C = FILTER B BY cuisines == 'pizza';" +
"D = STORE C INTO 'empty';" ;
LogicalPlan newLogicalPlan = buildPlan( query );
Operator load = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( load instanceof LOLoad );
Operator fe1 = newLogicalPlan.getSuccessors( load ).get( 0 );
Assert.assertTrue( fe1 instanceof LOForEach );
Operator fe2 = newLogicalPlan.getSuccessors( fe1 ).get( 0 );
Assert.assertTrue( fe2 instanceof LOForEach );
Operator filter = newLogicalPlan.getSuccessors( fe2 ).get( 0 );
Assert.assertTrue( filter instanceof LOFilter );
}
示例2: testSimple
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testSimple() throws Exception {
String query = "A =LOAD 'file.txt' AS (name, cuisines:bag{ t : ( cuisine ) } );" +
"B = FOREACH A GENERATE name, flatten(cuisines);" +
"C = FILTER B BY name == 'joe';" +
"D = STORE C INTO 'empty';" ;
LogicalPlan newLogicalPlan = buildPlan( query );
Operator load = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( load instanceof LOLoad );
Operator filter = newLogicalPlan.getSuccessors( load ).get( 0 );
Assert.assertTrue( filter instanceof LOFilter );
Operator fe1 = newLogicalPlan.getSuccessors( filter ).get( 0 );
Assert.assertTrue( fe1 instanceof LOForEach );
Operator fe2 = newLogicalPlan.getSuccessors( fe1 ).get( 0 );
Assert.assertTrue( fe2 instanceof LOForEach );
}
示例3: testFilterFilter
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testFilterFilter() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = filter A by $0 != 'name';" +
"C = filter B by $1 < 18;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
Assert.assertTrue( ((LOFilter)op).getAlias().equals( "B" ) );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
Assert.assertTrue( ((LOFilter)op).getAlias().equals( "C" ) );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
示例4: visit
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public void visit(LOSplit split) throws FrontendException {
List<Operator> branchOutputs = split.getPlan().getSuccessors(split);
for (int i=0;i<branchOutputs.size();i++) {
Operator branchOutput = branchOutputs.get(i);
Set<Long> branchOutputUids = (Set<Long>)branchOutput.getAnnotation(ColumnPruneHelper.INPUTUIDS);
if (branchOutputUids!=null) {
Set<Integer> columnsToDrop = new HashSet<Integer>();
for (int j=0;j<split.getSchema().size();j++) {
if (!branchOutputUids.contains(split.getSchema().getField(j).uid))
columnsToDrop.add(j);
}
if (!columnsToDrop.isEmpty()) {
LOForEach foreach = Util.addForEachAfter((LogicalPlan)split.getPlan(), split, i, columnsToDrop);
foreach.getSchema();
}
}
}
}
示例5: removeSubTree
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
private void removeSubTree(LogicalRelationalOperator op) throws FrontendException {
LogicalPlan p = (LogicalPlan)op.getPlan();
List<Operator> ll = p.getPredecessors(op);
if (ll != null) {
for(Operator pred: ll.toArray(new Operator[ll.size()])) {
removeSubTree((LogicalRelationalOperator)pred);
}
}
if (p.getSuccessors(op) != null) {
Operator[] succs = p.getSuccessors(op).toArray(new Operator[0]);
for(Operator s: succs) {
p.disconnect(op, s);
}
}
p.remove(op);
}
示例6: test3
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
/**
* No fields are used in filter condition at all.
* @throws Exception
*/
@Test
public void test3() throws Exception {
String query = "A =LOAD 'file.txt' AS (a:(u,v), b, c);" +
"B = FOREACH A GENERATE $0, b;" +
"C = FILTER B BY 8 > 5;" +
"STORE C INTO 'empty';";
LogicalPlan newLogicalPlan = buildPlan( query );
Operator load = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( load instanceof LOLoad );
Operator filter = newLogicalPlan.getSuccessors( load ).get( 0 );
Assert.assertTrue( filter instanceof LOFilter );
Operator fe1 = newLogicalPlan.getSuccessors( filter ).get( 0 );
Assert.assertTrue( fe1 instanceof LOForEach );
Operator fe2 = newLogicalPlan.getSuccessors( fe1 ).get( 0 );
Assert.assertTrue( fe2 instanceof LOForEach );
}
示例7: checkWarning
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
private void checkWarning(String query, String warnMsg) throws FrontendException {
LogicalPlan plan = generateLogicalPlan(query);
new ColumnAliasConversionVisitor( plan ).visit();
// validate
CompilationMessageCollector collector = new CompilationMessageCollector();
TypeCheckingRelVisitor typeChecker = new TypeCheckingRelVisitor(plan, collector);
typeChecker.visit();
new CastLineageSetter(plan, collector).visit();
printMessageCollector(collector);
boolean isWarningSeen = false;
assertTrue("message collector has message", collector.hasMessage());
for (Message msg : collector){
if (msg.getMessageType() == MessageType.Warning
&& msg.getMessage().contains(warnMsg)){
isWarningSeen = true;
}
}
assertTrue("Expected warning is seen", isWarningSeen);
}
示例8: testFilterAfterNestedLimit
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
/**
* Test that filter cannot get pushed up over nested Limit (see PIG-3347)
*/
@Test
public void testFilterAfterNestedLimit() throws Exception {
String query = "a = LOAD 'file.txt';" +
"a_group = group a by $0;" +
"b = foreach a_group { a_limit = limit a.$0 5;generate group, a_limit;}" +
"c = filter b by SIZE(a_limit) == 1;" +
"store c into 'empty';";
// filter should not be pushed above nested distinct,
//ie expect - loload -> locogroup -> foreach -> filter
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
newLogicalPlan.explain(System.out, "text", true);
Operator load = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( load instanceof LOLoad );
Operator cogroup = newLogicalPlan.getSuccessors( load ).get( 0 );
Assert.assertTrue( cogroup instanceof LOCogroup );
Operator foreach = newLogicalPlan.getSuccessors(cogroup).get( 0 );
Assert.assertTrue( foreach instanceof LOForEach );
Operator filter = newLogicalPlan.getSuccessors(foreach).get( 0 );
Assert.assertTrue( filter instanceof LOFilter );
}
示例9: testBuildLoadOpWithDefaultFunc
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testBuildLoadOpWithDefaultFunc() throws Exception {
String query = "a = load '1.txt';" +
"store a into 'output';";
LogicalPlan lp = buildPlan(query);
FuncSpec funcSpec = getFirstLoadFuncSpec(lp);
assertEquals("org.apache.pig.builtin.PigStorage", funcSpec.getClassName());
// set default load func in config
pigServer.getPigContext().getProperties().setProperty(PigConfiguration.PIG_DEFAULT_LOAD_FUNC, "org.apache.pig.test.PigStorageWithSchema");
query = "a = load '1.txt';" +
"store a into 'output';";
lp = buildPlan(query);
funcSpec = getFirstLoadFuncSpec(lp);
assertEquals("org.apache.pig.test.PigStorageWithSchema", funcSpec.getClassName());
// unset default load func
pigServer.getPigContext().getProperties().remove(PigConfiguration.PIG_DEFAULT_LOAD_FUNC);
}
示例10: testMultiQueryWithTwoStores
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testMultiQueryWithTwoStores() {
System.out.println("===== multi-query with 2 stores =====");
try {
myPig.setBatchOn();
myPig.registerQuery("a = load 'passwd' " +
"using PigStorage(':') as (uname:chararray, passwd:chararray, uid:int,gid:int);");
myPig.registerQuery("b = filter a by uid > 5;");
myPig.registerQuery("store b into '/tmp/output1';");
myPig.registerQuery("c = group b by gid;");
myPig.registerQuery("store c into '/tmp/output2';");
LogicalPlan lp = checkLogicalPlan(1, 2, 5);
PhysicalPlan pp = checkPhysicalPlan(lp, 1, 2, 11);
checkMRPlan(pp, 1, 1, 1);
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
}
}
示例11: testFilterConstantConditionGroupBy
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testFilterConstantConditionGroupBy() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = group A by $0;" +
"C = filter B by 1 == 1;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOForEach );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOCogroup );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
示例12: testNotPossibleFilter
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testNotPossibleFilter() throws Exception {
String query = "A =LOAD 'file.txt' AS (name, cuisines:bag{ t : ( cuisine ) } );" +
"B = FOREACH A GENERATE name, flatten(cuisines);" +
"C = FILTER B BY cuisine == 'French';" +
"D = STORE C INTO 'empty';";
LogicalPlan newLogicalPlan = buildPlan( query );
Operator load = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( load instanceof LOLoad );
Operator fe1 = newLogicalPlan.getSuccessors( load ).get( 0 );
Assert.assertTrue( fe1 instanceof LOForEach );
Operator fe2 = newLogicalPlan.getSuccessors( fe1 ).get( 0 );
Assert.assertTrue( fe2 instanceof LOForEach );
Operator filter = newLogicalPlan.getSuccessors( fe2 ).get( 0 );
Assert.assertTrue( filter instanceof LOFilter );
}
示例13: testFilterConstantConditionGroupBy
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
@Test
public void testFilterConstantConditionGroupBy() throws Exception {
String query = "A = load 'myfile' as (name, age, gpa);" +
"B = group A by $0;" +
"C = filter B by 1 == 1;" +
"D = STORE C into 'dummy';";
LogicalPlan newLogicalPlan = migrateAndOptimizePlan( query );
Operator op = newLogicalPlan.getSources().get( 0 );
Assert.assertTrue( op instanceof LOLoad );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOFilter );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOCogroup );
op = newLogicalPlan.getSuccessors(op).get( 0 );
Assert.assertTrue( op instanceof LOStore );
}
示例14: createDummyRelOpWithAlias
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
/**
* @return a dummy logical relational operator
*/
private LogicalRelationalOperator createDummyRelOpWithAlias() {
class DummyRelOp extends LogicalRelationalOperator{
DummyRelOp(){
super("dummy", new LogicalPlan());
this.alias = "dummy";
}
@Override
public LogicalSchema getSchema() throws FrontendException {
return null;
}
@Override
public void accept(PlanVisitor v) throws FrontendException {
}
@Override
public boolean isEqual(Operator operator) throws FrontendException {
return false;
}
}
return new DummyRelOp();
}
示例15: attachStorePlan
import org.apache.pig.newplan.logical.relational.LogicalPlan; //导入依赖的package包/类
public static void attachStorePlan(String scope, LogicalPlan lp, String fileName, String func,
Operator input, String alias, PigContext pigContext) throws FrontendException {
func = func == null ? pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName()) : func;
FuncSpec funcSpec = new FuncSpec( func );
StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec( funcSpec );
fileName = removeQuotes( fileName );
FileSpec fileSpec = new FileSpec( fileName, funcSpec );
String sig = alias + "_" + LogicalPlanBuilder.newOperatorKey(scope);
stoFunc.setStoreFuncUDFContextSignature(sig);
LOStore store = new LOStore(lp, fileSpec, stoFunc, sig);
store.setAlias(alias);
try {
stoFunc.relToAbsPathForStoreLocation( fileName, getCurrentDir( pigContext ) );
} catch (IOException ioe) {
FrontendException e = new FrontendException( ioe.getMessage(), ioe );
throw e;
}
lp.add( store );
lp.connect( input, store );
}