Java PColFilterExtractor类代码示例

import org.apache.pig.newplan.PColFilterExtractor; //导入依赖的package包/类
private void negativeTest(String query, List<String> partitionCols,
        int expectedErrorCode) throws Exception {
    PigServer pigServer = new PigServer( pc );
    LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);
    Operator op = newLogicalPlan.getSinks().get(0);
    LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
    PColFilterExtractor pColExtractor = new PColFilterExtractor(
            filter.getFilterPlan(), partitionCols);
    try {
        pColExtractor.visit();
    } catch(Exception e) {
        Assert.assertEquals("Checking if exception has right error code",
                expectedErrorCode, LogUtils.getPigException(e).getErrorCode());
        return;
    }
}
 

import org.apache.pig.newplan.PColFilterExtractor; //导入依赖的package包/类
private void negativeTest(String query, List<String> partitionCols,
        int expectedErrorCode) throws Exception {
    PigServer pigServer = new PigServer( pc );
    LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);
    Operator op = newLogicalPlan.getSinks().get(0);
    LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
    PColFilterExtractor pColExtractor = new PColFilterExtractor(
            filter.getFilterPlan(), partitionCols);
    try {
        pColExtractor.visit();
    } catch(Exception e) {
        Assert.assertEquals("Checking if exception has right error code", 
                expectedErrorCode, LogUtils.getPigException(e).getErrorCode());
        return;
    }
}
 

import org.apache.pig.newplan.PColFilterExtractor; //导入依赖的package包/类
@Override
    public void transform(OperatorPlan matched) throws FrontendException {
    	subPlan = new OperatorSubPlan( currentPlan );

    	setupColNameMaps();
    	
    	// PIG-1871: Don't throw exception if partition filters cannot be pushed up. 
    	// Perform transformation on a copy of the filter plan, and replace the 
    	// original filter plan only if the transformation is successful 
    	// (i.e. partition filter can be pushed down) 
    	LogicalExpressionPlan filterExpr = loFilter.getFilterPlan();
    	LogicalExpressionPlan filterExprCopy = filterExpr.deepCopy();
    	
    	PColFilterExtractor pColFilterFinder = new PColFilterExtractor(
    	        filterExprCopy, getMappedKeys( partitionKeys ) );
    	pColFilterFinder.visit();
    	Expression partitionFilter = pColFilterFinder.getPColCondition();
    	
    	if(partitionFilter != null) {
    		// the column names in the filter may be the ones provided by
    		// the user in the schema in the load statement - we may need
    		// to replace them with partition column names as given by
    		// LoadFunc.getSchema()
    		updateMappedColNames(partitionFilter);
    		try {
	loadMetadata.setPartitionFilter(partitionFilter);
} catch (IOException e) {
	throw new FrontendException( e );
}
    		if(pColFilterFinder.isFilterRemovable()) {  
    			currentPlan.removeAndReconnect( loFilter );
    		} else {
    		    loFilter.setFilterPlan(filterExprCopy);
    		}
    	}
    }
 

import org.apache.pig.newplan.PColFilterExtractor; //导入依赖的package包/类
/**
 * Test that pig sends correct partition column names in setPartitionFilter
 * when the user has a schema in the load statement which renames partition
 * columns
 * @throws Exception
 */
@Test
public void testColNameMapping1() throws Exception {
    TestLoader.partFilter = null;
    String q = "a = load 'foo' using "
        + TestLoader.class.getName() +
        "('srcid:int, mrkt:chararray, dstid:int, name:chararray, age:int', " +
        "'srcid,mrkt') as (f1, f2, f3, f4, f5);" +
        "b = filter a by " +
        "(f5 >= 20 and f2 == 'us') and (f1 == 10 and f3 == 15);" +
        "store b into 'out';";

    LogicalPlan newLogicalPlan = migrateAndOptimizePlan( q );

    Assert.assertEquals("checking partition filter:",
            "((mrkt == 'us') and (srcid == 10))",
            TestLoader.partFilter.toString());
    Operator op = newLogicalPlan.getSinks().get(0);
    LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);

    PColFilterExtractor extractor = new PColFilterExtractor(filter.getFilterPlan(), new ArrayList<String>());

    String actual = extractor.getExpression(
            (LogicalExpression)filter.getFilterPlan().getSources().get(0)).
            toString().toLowerCase();
    Assert.assertEquals("checking trimmed filter expression:",
            "((f5 >= 20) and (f3 == 15))", actual);
}
 

import org.apache.pig.newplan.PColFilterExtractor; //导入依赖的package包/类
/**
 * Test that pig sends correct partition column names in setPartitionFilter
 * when the user has a schema in the load statement which renames partition
 * columns - in this test case there is no condition on partition columns
 * - so setPartitionFilter() should not be called and the filter condition
 * should remain as is.
 * @throws Exception
 */
@Test
public void testColNameMapping2() throws Exception {
    TestLoader.partFilter = null;
    String q = "a = load 'foo' using "
        + TestLoader.class.getName() +
        "('srcid:int, mrkt:chararray, dstid:int, name:chararray, age:int', " +
        "'srcid') as (f1, f2, f3, f4, f5);" +
        "b = filter a by " +
        "f5 >= 20 and f2 == 'us' and f3 == 15;" +
        "store b into 'out';";

    LogicalPlan newLogicalPlan = migrateAndOptimizePlan( q );

    Assert.assertEquals("checking partition filter:",
            null,
            TestLoader.partFilter);
    Operator op = newLogicalPlan.getSinks().get(0);
    LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);

    PColFilterExtractor extractor = new PColFilterExtractor(filter.getFilterPlan(), new ArrayList<String>());

    String actual = extractor.getExpression(
            (LogicalExpression) filter.getFilterPlan().
            getSources().get(0)).
            toString().toLowerCase();
    Assert.assertEquals("checking trimmed filter expression:",
            "(((f5 >= 20) and (f2 == 'us')) and (f3 == 15))", actual);
}
 

import org.apache.pig.newplan.PColFilterExtractor; //导入依赖的package包/类
/**
 * Test that pig sends correct partition column names in setPartitionFilter
 * when the user has a schema in the load statement which renames partition
 * columns - in this test case the schema in load statement is a prefix
 * (with columns renamed) of the schema returned by
 * {@link LoadMetadata#getSchema(String, Configuration)}
 * @throws Exception
 */
@Test
public void testColNameMapping4() throws Exception {
    TestLoader.partFilter = null;
    String q = "a = load 'foo' using "
        + TestLoader.class.getName() +
        "('srcid:int, mrkt:chararray, dstid:int, name:chararray, age:int', " +
        "'srcid,mrkt') as (f1:int, f2:chararray, f3:int, name:chararray, age:int);" +
        "b = filter a by " +
        "(age >= 20 and f2 == 'us') and (f1 == 10 and f3 == 15);" + "store b into 'out';";

    LogicalPlan newLogicalPlan = migrateAndOptimizePlan( q );

    Assert.assertEquals("checking partition filter:",
            "((mrkt == 'us') and (srcid == 10))",
            TestLoader.partFilter.toString());
    Operator op = newLogicalPlan.getSinks().get(0);
    LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);

    PColFilterExtractor extractor = new PColFilterExtractor(filter.getFilterPlan(), new ArrayList<String>());

    String actual = extractor.getExpression(
            (LogicalExpression) filter.getFilterPlan().getSources().get(0)).
            toString().toLowerCase();
    Assert.assertEquals("checking trimmed filter expression:",
            "((age >= 20) and (f3 == 15))", actual);
}
 

import org.apache.pig.newplan.PColFilterExtractor; //导入依赖的package包/类
private PColFilterExtractor test(String query, List<String> partitionCols,
        String expPartFilterString, String expFilterString, boolean skipTrimmedFilterCheck)
throws Exception {
    PigServer pigServer = new PigServer( pc );
    LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);
    Operator op = newLogicalPlan.getSinks().get(0);
    LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
    PColFilterExtractor pColExtractor = new PColFilterExtractor(
            filter.getFilterPlan(), partitionCols);
    pColExtractor.visit();

    if(expPartFilterString == null) {
        Assert.assertEquals("Checking partition column filter:", null,
                pColExtractor.getPColCondition());
    } else  {
        Assert.assertEquals("Checking partition column filter:",
                expPartFilterString,
                pColExtractor.getPColCondition().toString());
    }

    // The getExpression() in PColFilterExtractor was written to get the
    // pushdown filter expression and does not have support for columns of
    // type tuple or map as partition columns are expected to be of
    // primitive data type. But we are using the method in the tests for forming
    // trimmed filter after pushdown. So skip check in cases where we expect a
    // trimmed filter to have a map or tuple in the condition.
    if (!skipTrimmedFilterCheck) {
        if (expFilterString == null) {
            Assert.assertTrue("Check that filter can be removed:",
                    pColExtractor.isFilterRemovable());
        } else {
            String actual = pColExtractor
                    .getExpression(
                            (LogicalExpression) filter.getFilterPlan().getSources().get(0))
                    .toString();
            Assert.assertEquals("checking trimmed filter expression:", expFilterString, actual);
        }
    }
    return pColExtractor;
}
 

import org.apache.pig.newplan.PColFilterExtractor; //导入依赖的package包/类
/**
 * Test that pig sends correct partition column names in setPartitionFilter
 * when the user has a schema in the load statement which renames partition
 * columns
 * @throws Exception
 */
@Test
public void testColNameMapping1() throws Exception {
    TestLoader.partFilter = null;
    String q = "a = load 'foo' using "
        + TestLoader.class.getName() + 
        "('srcid:int, mrkt:chararray, dstid:int, name:chararray, age:int', " +
        "'srcid,mrkt') as (f1, f2, f3, f4, f5);" +
        "b = filter a by " +
        "(f5 >= 20 and f2 == 'us') and (f1 == 10 and f3 == 15);" + 
        "store b into 'out';";

    LogicalPlan newLogicalPlan = migrateAndOptimizePlan( q );

    Assert.assertEquals("checking partition filter:",             
            "((mrkt == 'us') and (srcid == 10))",
            TestLoader.partFilter.toString());
    Operator op = newLogicalPlan.getSinks().get(0);
    LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
    
    PColFilterExtractor extractor = new PColFilterExtractor(filter.getFilterPlan(), new ArrayList<String>());
    
    String actual = extractor.getExpression(
            (LogicalExpression)filter.getFilterPlan().getSources().get(0)).
            toString().toLowerCase();
    Assert.assertEquals("checking trimmed filter expression:", 
            "((f5 >= 20) and (f3 == 15))", actual);
}
 

import org.apache.pig.newplan.PColFilterExtractor; //导入依赖的package包/类
/**
 * Test that pig sends correct partition column names in setPartitionFilter
 * when the user has a schema in the load statement which renames partition
 * columns - in this test case there is no condition on partition columns
 * - so setPartitionFilter() should not be called and the filter condition
 * should remain as is.
 * @throws Exception
 */
@Test
public void testColNameMapping2() throws Exception {
    TestLoader.partFilter = null;
    String q = "a = load 'foo' using "
        + TestLoader.class.getName() + 
        "('srcid:int, mrkt:chararray, dstid:int, name:chararray, age:int', " +
        "'srcid') as (f1, f2, f3, f4, f5);" +
        "b = filter a by " +
        "f5 >= 20 and f2 == 'us' and f3 == 15;" +
        "store b into 'out';";

    LogicalPlan newLogicalPlan = migrateAndOptimizePlan( q );

    Assert.assertEquals("checking partition filter:",             
            null,
            TestLoader.partFilter);
    Operator op = newLogicalPlan.getSinks().get(0);
    LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
    
    PColFilterExtractor extractor = new PColFilterExtractor(filter.getFilterPlan(), new ArrayList<String>());
    
    String actual = extractor.getExpression(
            (LogicalExpression) filter.getFilterPlan().
            getSources().get(0)).
            toString().toLowerCase();
    Assert.assertEquals("checking trimmed filter expression:", 
            "(((f5 >= 20) and (f2 == 'us')) and (f3 == 15))", actual);
}
 

import org.apache.pig.newplan.PColFilterExtractor; //导入依赖的package包/类
/**
 * Test that pig sends correct partition column names in setPartitionFilter
 * when the user has a schema in the load statement which renames partition
 * columns - in this test case the schema in load statement is a prefix 
 * (with columns renamed) of the schema returned by 
 * {@link LoadMetadata#getSchema(String, Configuration)}
 * @throws Exception
 */
@Test
public void testColNameMapping4() throws Exception {
    TestLoader.partFilter = null;
    String q = "a = load 'foo' using "
        + TestLoader.class.getName() + 
        "('srcid:int, mrkt:chararray, dstid:int, name:chararray, age:int', " +
        "'srcid,mrkt') as (f1:int, f2:chararray, f3:int, name:chararray, age:int);" +
        "b = filter a by " +
        "(age >= 20 and f2 == 'us') and (f1 == 10 and f3 == 15);" + "store b into 'out';";

    LogicalPlan newLogicalPlan = migrateAndOptimizePlan( q );

    Assert.assertEquals("checking partition filter:",             
            "((mrkt == 'us') and (srcid == 10))",
            TestLoader.partFilter.toString());
    Operator op = newLogicalPlan.getSinks().get(0);
    LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
    
    PColFilterExtractor extractor = new PColFilterExtractor(filter.getFilterPlan(), new ArrayList<String>());
    
    String actual = extractor.getExpression(
            (LogicalExpression) filter.getFilterPlan().getSources().get(0)).
            toString().toLowerCase();
    Assert.assertEquals("checking trimmed filter expression:", 
            "((age >= 20) and (f3 == 15))", actual);
}
 

import org.apache.pig.newplan.PColFilterExtractor; //导入依赖的package包/类
private PColFilterExtractor test(String query, List<String> partitionCols, 
        String expPartFilterString, String expFilterString) 
throws Exception {
    PigServer pigServer = new PigServer( pc );
    LogicalPlan newLogicalPlan = Util.buildLp(pigServer, query);
    Operator op = newLogicalPlan.getSinks().get(0);
    LOFilter filter = (LOFilter)newLogicalPlan.getPredecessors(op).get(0);
    PColFilterExtractor pColExtractor = new PColFilterExtractor(
            filter.getFilterPlan(), partitionCols);
    pColExtractor.visit();

    if(expPartFilterString == null) {
        Assert.assertEquals("Checking partition column filter:", null, 
                pColExtractor.getPColCondition());
    } else  {
        Assert.assertEquals("Checking partition column filter:", 
                expPartFilterString.toLowerCase(), 
                pColExtractor.getPColCondition().toString().toLowerCase());   
    }

    if(expFilterString == null) {
        Assert.assertTrue("Check that filter can be removed:", 
                pColExtractor.isFilterRemovable());
    } else {
        String actual = pColExtractor.getExpression(
                (LogicalExpression)filter.getFilterPlan().getSources().get(0)).
                toString().toLowerCase();
        Assert.assertEquals("checking trimmed filter expression:", expFilterString,
                actual);
    }
    return pColExtractor;
}