本文整理汇总了Java中org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer类的典型用法代码示例。如果您正苦于以下问题：Java SecondaryKeyOptimizer类的具体用法？Java SecondaryKeyOptimizer怎么用？Java SecondaryKeyOptimizer使用的例子？那么, 这里精选的类代码示例或许可以为您提供帮助。
SecondaryKeyOptimizer类属于org.apache.pig.backend.hadoop.executionengine.mapReduceLayer包，在下文中一共展示了SecondaryKeyOptimizer类的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: testDistinctOptimization1
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer; //导入依赖的package包/类
@Test
public void testDistinctOptimization1() throws Exception {
// Distinct on one entire input
String query = ("A=LOAD 'input1' AS (a0, a1, a2);"+
"B = group A by $0;"+
"C = foreach B { D = distinct A; generate group, D;};"+
"store C into 'output';");
PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
so.visit();
assertEquals(1, so.getNumMRUseSecondaryKey());
assertEquals(0, so.getNumSortRemoved());
assertEquals(1, so.getDistinctChanged());
}
示例2: testSortOptimization1
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer; //导入依赖的package包/类
@Test
public void testSortOptimization1() throws Exception {
// Sort on something other than the main key
String query = ("A=LOAD 'input1' AS (a0, a1, a2);" +
"B = group A by $0;" +
"C = foreach B { D = limit A 10; E = order D by $1; generate group, E;};" +
"store C into 'output';");
PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
so.visit();
assertEquals(1, so.getNumMRUseSecondaryKey());
assertEquals(1, so.getNumSortRemoved());
assertEquals(0, so.getDistinctChanged());
}
示例3: testSortOptimization2
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer; //导入依赖的package包/类
@Test
public void testSortOptimization2() throws Exception {
// Sort on the prefix of the main key
String query = ("A=LOAD 'input1' AS (a0, a1, a2);" +
"B = group A by $0;" +
"C = foreach B { D = limit A 10; E = order D by $0; generate group, E;};" +
"store C into 'output';");
PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
so.visit();
assertEquals(0, so.getNumMRUseSecondaryKey());
assertEquals(1, so.getNumSortRemoved());
assertEquals(0, so.getDistinctChanged());
}
示例4: testSortOptimization3
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer; //导入依赖的package包/类
@Test
public void testSortOptimization3() throws Exception {
// Sort on the main key prefix / non main key prefix mixed
String query = ("A=LOAD 'input1' AS (a0, a1, a2);" +
"B = group A by $0;" +
"C = foreach B { D = limit A 10; E = order D by $1; F = order E by $0; generate group, F;};"+
"store C into 'output';");
PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
so.visit();
assertEquals(1, so.getNumMRUseSecondaryKey());
assertEquals(2, so.getNumSortRemoved());
assertEquals(0, so.getDistinctChanged());
}
示例5: testSortOptimization4
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer; //导入依赖的package包/类
@Test
public void testSortOptimization4() throws Exception {
// Sort on the main key again
String query = ("A=LOAD 'input1' AS (a0, a1, a2);" +
"B = group A by $0;" +
"C = foreach B { D = limit A 10; E = order D by $0, $1, $2; generate group, E;};" +
"store C into 'output';");
PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
so.visit();
assertEquals(1, so.getNumMRUseSecondaryKey());
assertEquals(1, so.getNumSortRemoved());
assertEquals(0, so.getDistinctChanged());
}
示例6: testSortOptimization5
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer; //导入依赖的package包/类
@Test
public void testSortOptimization5() throws Exception {
// Sort on the two keys, we can only take off 1
String query = ("A=LOAD 'input1' AS (a0, a1, a2);" +
"B = group A by $0;" +
"C = foreach B { D = limit A 10; E = order D by $1; F = order E by $2; generate group, F;};" +
"store C into 'output';");
PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
so.visit();
assertEquals(1, so.getNumMRUseSecondaryKey());
assertEquals(1, so.getNumSortRemoved());
assertEquals(0, so.getDistinctChanged());
}
示例7: testSortOptimization6
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer; //导入依赖的package包/类
@Test
public void testSortOptimization6() throws Exception {
// Sort desc
String query = ("A=LOAD 'input1' AS (a0, a1, a2);" +
"B = group A by $0;" +
"C = foreach B { D = order A by $0 desc; generate group, D;};" +
"store C into 'output';");
PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
so.visit();
assertEquals(1, so.getNumMRUseSecondaryKey());
assertEquals(1, so.getNumSortRemoved());
assertEquals(0, so.getDistinctChanged());
}
示例8: testSortOptimization7
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer; //导入依赖的package包/类
@Test
public void testSortOptimization7() throws Exception {
// Sort asc on 1st key, desc on 2nd key
String query = ("A=LOAD 'input1' AS (a0, a1, a2);" +
"B = group A by ($0, $1);" +
"C = foreach B { D = order A by $0, $1 desc; generate group, D;};" +
"store C into 'output';");
PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
so.visit();
assertEquals(1, so.getNumMRUseSecondaryKey());
assertEquals(1, so.getNumSortRemoved());
assertEquals(0, so.getDistinctChanged());
}
示例9: testSortOptimization8
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer; //导入依赖的package包/类
@Test
public void testSortOptimization8() throws Exception {
// Sort desc, used in UDF twice
String query = ("A=LOAD 'input1' AS (a0);" +
"B = group A all;" +
"C = foreach B { D = order A by $0 desc; generate DIFF(D, D);};" +
"store C into 'output';");
PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
so.visit();
assertEquals(1, so.getNumMRUseSecondaryKey());
assertEquals(2, so.getNumSortRemoved());
assertEquals(0, so.getDistinctChanged());
}
示例10: testDistinctOptimization2
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer; //导入依赖的package包/类
@Test
public void testDistinctOptimization2() throws Exception {
// Distinct on one entire input
String query = ("A=LOAD 'input1' AS (a0, a1, a2);"+
"B = group A by $0;"+
"C = foreach B { D = distinct A; generate group, D;};"+
"store C into 'output';");
PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
so.visit();
assertTrue(so.getNumMRUseSecondaryKey() == 1);
assertTrue(so.getNumSortRemoved() == 0);
assertTrue(so.getDistinctChanged() == 1);
}
示例11: testDistinctOptimization3
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer; //导入依赖的package包/类
@Test
public void testDistinctOptimization3() throws Exception {
// Distinct on the prefix of main sort key
String query = ("A=LOAD 'input1' AS (a0, a1, a2);"+
"B = group A by $0;"+
"C = foreach B { D = A.a0; E = distinct D; generate group, E;};"+
"store C into 'output';");
PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
so.visit();
assertTrue(so.getNumMRUseSecondaryKey() == 0);
assertTrue(so.getNumSortRemoved() == 0);
assertTrue(so.getDistinctChanged() == 1);
}
示例12: testDistinctOptimization4
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer; //导入依赖的package包/类
@Test
public void testDistinctOptimization4() throws Exception {
// Distinct on secondary key again, should remove
String query = ("A=LOAD 'input1' AS (a0, a1, a2);"+
"B = group A by $0;"+
"C = foreach B { D = A.a1; E = distinct D; F = distinct E; generate group, F;};"+
"store C into 'output';");
PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
so.visit();
assertTrue(so.getNumMRUseSecondaryKey() == 1);
assertTrue(so.getNumSortRemoved() == 0);
assertTrue(so.getDistinctChanged() == 2);
}
示例13: testDistinctOptimization5
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer; //导入依赖的package包/类
@Test
public void testDistinctOptimization5() throws Exception {
// Filter in foreach plan
String query = ("A=LOAD 'input1' AS (a0, a1, a2);" +
"B = group A by $0;" +
"C = foreach B { D = A.a1; E = distinct D; F = filter E by $0=='1'; generate group, F;};" +
"store C into 'output';");
PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
so.visit();
assertTrue(so.getNumMRUseSecondaryKey() == 1);
assertTrue(so.getNumSortRemoved() == 0);
assertTrue(so.getDistinctChanged() == 1);
}
示例14: testDistinctOptimization6
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer; //导入依赖的package包/类
@Test
public void testDistinctOptimization6() throws Exception {
// group by * with no schema, and distinct key is not part of main key
String query = ("A=LOAD 'input1';" +
"B = group A by *;" +
"C = foreach B { D = limit A 10; E = D.$1; F = DISTINCT E; generate group, COUNT(F);};" +
"store C into 'output';");
PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
so.visit();
assertTrue(so.getNumMRUseSecondaryKey() == 1);
assertTrue(so.getNumSortRemoved() == 0);
assertTrue(so.getDistinctChanged() == 1);
}
示例15: testDistinctOptimization7
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SecondaryKeyOptimizer; //导入依赖的package包/类
@Test
public void testDistinctOptimization7() throws Exception {
// group by * with no schema, distinct key is more specific than the main key
String query = ("A=LOAD 'input1';" +
"B = group A by *;" +
"C = foreach B { D = limit A 10; E = D.$0; F = DISTINCT E; generate group, COUNT(F);};" +
"store C into 'output';");
PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
SecondaryKeyOptimizer so = new SecondaryKeyOptimizer(mrPlan);
so.visit();
assertTrue(so.getNumMRUseSecondaryKey() == 1);
assertTrue(so.getNumSortRemoved() == 0);
assertTrue(so.getDistinctChanged() == 1);
}