本文整理汇总了Java中org.apache.pig.builtin.PigStorage类的典型用法代码示例。如果您正苦于以下问题：Java PigStorage类的具体用法？Java PigStorage怎么用？Java PigStorage使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
PigStorage类属于org.apache.pig.builtin包，在下文中一共展示了PigStorage类的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: attachStorePlan
import org.apache.pig.builtin.PigStorage; //导入依赖的package包/类
public static void attachStorePlan(String scope, LogicalPlan lp, String fileName, String func,
Operator input, String alias, PigContext pigContext) throws FrontendException {
func = func == null ? pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName()) : func;
FuncSpec funcSpec = new FuncSpec( func );
StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec( funcSpec );
fileName = removeQuotes( fileName );
FileSpec fileSpec = new FileSpec( fileName, funcSpec );
String sig = alias + "_" + LogicalPlanBuilder.newOperatorKey(scope);
stoFunc.setStoreFuncUDFContextSignature(sig);
LOStore store = new LOStore(lp, fileSpec, stoFunc, sig);
store.setAlias(alias);
try {
stoFunc.relToAbsPathForStoreLocation( fileName, getCurrentDir( pigContext ) );
} catch (IOException ioe) {
FrontendException e = new FrontendException( ioe.getMessage(), ioe );
throw e;
}
lp.add( store );
lp.connect( input, store );
}
示例2: testFinishInReduceMR
import org.apache.pig.builtin.PigStorage; //导入依赖的package包/类
@Test
public void testFinishInReduceMR() throws Exception {
String inputFileName = setUp(ExecType.MAPREDUCE);
// this file will be created on the cluster if finish() is called
String expectedFileName = "testFinishInReduceMR-finish.txt";
pigServer.registerQuery("define MYUDF " + MyEvalFunction.class.getName() + "('MAPREDUCE','"
+ expectedFileName + "');");
pigServer.registerQuery("a = load '" + inputFileName + "' using "
+ PigStorage.class.getName() + "(':');");
pigServer.registerQuery("a1 = group a by $1;");
pigServer.registerQuery("b = foreach a1 generate MYUDF" + "(*);");
Iterator<Tuple> iter = pigServer.openIterator("b");
while (iter.hasNext()) {
iter.next();
}
checkAndCleanup(ExecType.MAPREDUCE, expectedFileName, inputFileName);
}
示例3: testFunctionInsideFunction
import org.apache.pig.builtin.PigStorage; //导入依赖的package包/类
@Test
public void testFunctionInsideFunction() throws Exception{
File f1 = createFile(new String[]{"a:1","b:1","a:1"});
pigServer.registerQuery("a = load '"
+ Util.generateURI(Util.encodeEscape(f1.toString()), pigServer.getPigContext())
+ "' using " + PigStorage.class.getName() + "(':');");
pigServer.registerQuery("b = foreach a generate 1-1/1;");
Iterator<Tuple> iter = pigServer.openIterator("b");
for (int i=0 ;i<3; i++){
Assert.assertEquals(DataType.toDouble(iter.next().get(0)), 0.0);
}
}
示例4: testFinishInMapMR
import org.apache.pig.builtin.PigStorage; //导入依赖的package包/类
@Test
public void testFinishInMapMR() throws Exception {
String inputFileName = setUp(cluster.getExecType());
// this file will be created on the cluster if finish() is called
String expectedFileName = "testFinishInMapMR-finish.txt";
pigServer.registerQuery("define MYUDF " + MyEvalFunction.class.getName() + "('MAPREDUCE','"
+ expectedFileName + "');");
pigServer.registerQuery("a = load '" + Util.encodeEscape(inputFileName) + "' using "
+ PigStorage.class.getName() + "(':');");
pigServer.registerQuery("b = foreach a generate MYUDF" + "(*);");
Iterator<Tuple> iter = pigServer.openIterator("b");
while (iter.hasNext()) {
iter.next();
}
checkAndCleanup(cluster.getExecType(), expectedFileName, inputFileName);
}
示例5: testJoin
import org.apache.pig.builtin.PigStorage; //导入依赖的package包/类
@Test
public void testJoin() throws Exception{
File f1 = createFile(new String[]{"a:1","b:1","a:1"});
File f2 = createFile(new String[]{"b","b","a"});
pigServer.registerQuery("a = load '"
+ Util.generateURI(f1.toString(), pigServer.getPigContext())
+ "' using " + PigStorage.class.getName() + "(':');");
pigServer.registerQuery("b = load '"
+ Util.generateURI(f2.toString(), pigServer.getPigContext())
+ "';");
pigServer.registerQuery("c = cogroup a by $0, b by $0;");
pigServer.registerQuery("d = foreach c generate flatten($1),flatten($2);");
Iterator<Tuple> iter = pigServer.openIterator("d");
int count = 0;
while(iter.hasNext()){
Tuple t = iter.next();
Assert.assertTrue(t.get(0).toString().equals(t.get(2).toString()));
count++;
}
Assert.assertEquals(count, 4);
}
示例6: testFunctionInsideFunction
import org.apache.pig.builtin.PigStorage; //导入依赖的package包/类
@Test
public void testFunctionInsideFunction() throws Exception{
File f1 = createFile(new String[]{"a:1","b:1","a:1"});
pigServer.registerQuery("a = load '"
+ Util.generateURI(f1.toString(), pigServer.getPigContext())
+ "' using " + PigStorage.class.getName() + "(':');");
pigServer.registerQuery("b = foreach a generate 1-1/1;");
Iterator<Tuple> iter = pigServer.openIterator("b");
for (int i=0 ;i<3; i++){
Assert.assertEquals(DataType.toDouble(iter.next().get(0)), 0.0);
}
}
示例7: testLimitStoreSchema1
import org.apache.pig.builtin.PigStorage; //导入依赖的package包/类
@Test //end to end test
public void testLimitStoreSchema1() throws Exception{
Util.createLocalInputFile("student", new String[]{"joe smith:18:3.5","amy brown:25:2.5","jim fox:20:4.0","leo fu:55:3.0"});
pigServer.registerQuery("a = load 'student' using " + PigStorage.class.getName() + "(':') as (name, age, gpa);");
pigServer.registerQuery("d = distinct a;");
pigServer.registerQuery("lim = limit d 1;");
String outFile = "limitSchemaOut";
Util.deleteDirectory(new File(outFile));
pigServer.store("lim", outFile, "PigStorage('\\t', '-schema')");
pigServer.dumpSchema("lim");
pigServer.registerQuery("b = LOAD '" + outFile + "' using PigStorage('\\t', '-schema');");
Schema genSchema = pigServer.dumpSchema("b");
System.err.println(genSchema);
Assert.assertNotNull(genSchema);
}
示例8: testQueryParser
import org.apache.pig.builtin.PigStorage; //导入依赖的package包/类
@Test
public void testQueryParser() throws Exception{
File f1 = File.createTempFile("tmp", "");
PrintWriter pw = new PrintWriter(f1, "UTF-8");
pw.println("中文");
pw.close();
pigServer.registerQuery("a = load '"
+ Util.encodeEscape(Util.generateURI(f1.toString(), pigServer.getPigContext()))
+ "' using " + PigStorage.class.getName() + "();");
pigServer.registerQuery("b = filter a by $0 == '中文';");
Iterator<Tuple> iter = pigServer.openIterator("a");
assertEquals(DataType.toString(iter.next().get(0)), "中文");
f1.delete();
}
示例9: testJoin
import org.apache.pig.builtin.PigStorage; //导入依赖的package包/类
@Test
public void testJoin() throws Exception{
File f1 = Util.createFile(new String[]{"a:1","b:1","a:1"});
File f2 = Util.createFile(new String[]{"b","b","a"});
pigServer.registerQuery("a = load '"
+ Util.generateURI(f1.toString(), pigContext) + "' using "
+ PigStorage.class.getName() + "(':');");
pigServer.registerQuery("b = load '"
+ Util.generateURI(f2.toString(), pigContext) + "';");
pigServer.registerQuery("c = cogroup a by $0, b by $0;");
pigServer.registerQuery("d = foreach c generate flatten($1),flatten($2);");
Iterator<Tuple> iter = pigServer.openIterator("d");
int count = 0;
while(iter.hasNext()){
Tuple t = iter.next();
Assert.assertTrue(t.get(0).toString().equals(t.get(2).toString()));
count++;
}
Assert.assertEquals(count, 4);
}
示例10: testGetInputSizeFromFs
import org.apache.pig.builtin.PigStorage; //导入依赖的package包/类
@Test
public void testGetInputSizeFromFs() throws Exception {
long size = 2L * 1024 * 1024 * 1024;
Assert.assertEquals(size, InputSizeReducerEstimator.getTotalInputFileSize(
CONF, Lists.newArrayList(createPOLoadWithSize(size, new PigStorage())),
new org.apache.hadoop.mapreduce.Job(CONF)));
Assert.assertEquals(size, InputSizeReducerEstimator.getTotalInputFileSize(
CONF,
Lists.newArrayList(createPOLoadWithSize(size, new PigStorageWithStatistics())),
new org.apache.hadoop.mapreduce.Job(CONF)));
Assert.assertEquals(size * 2, InputSizeReducerEstimator.getTotalInputFileSize(
CONF,
Lists.newArrayList(
createPOLoadWithSize(size, new PigStorage()),
createPOLoadWithSize(size, new PigStorageWithStatistics())),
new org.apache.hadoop.mapreduce.Job(CONF)));
}
示例11: attachStorePlan
import org.apache.pig.builtin.PigStorage; //导入依赖的package包/类
public static void attachStorePlan(String scope, LogicalPlan lp, String fileName, String func,
Operator input, String alias, PigContext pigContext) throws FrontendException {
func = func == null ? pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName()) : func;
FuncSpec funcSpec = new FuncSpec( func );
StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec( funcSpec );
fileName = removeQuotes( fileName );
FileSpec fileSpec = new FileSpec( fileName, funcSpec );
String sig = alias + "_" + LogicalPlanBuilder.newOperatorKey(scope);
stoFunc.setStoreFuncUDFContextSignature(sig);
LOStore store = new LOStore(lp, fileSpec, stoFunc, sig);
store.setAlias(alias);
try {
stoFunc.relToAbsPathForStoreLocation( fileName, getCurrentDir( pigContext ) );
} catch (IOException ioe) {
FrontendException e = new FrontendException( ioe.getMessage(), ioe );
throw e;
}
lp.add( store );
lp.connect( input, store );
}
示例12: buildStoreOp
import org.apache.pig.builtin.PigStorage; //导入依赖的package包/类
String buildStoreOp(SourceLocation loc, String alias, String inputAlias, String filename, FuncSpec funcSpec)
throws ParserValidationException {
try {
// Load StoreFunc class from default properties if funcSpec is null. Fallback on PigStorage if StoreFunc is not specified in properties.
funcSpec = funcSpec == null ? new FuncSpec(pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName())) : funcSpec;
StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(funcSpec);
String fileNameKey = inputAlias + "_" + (storeIndex++) ;
String signature = inputAlias + "_" + newOperatorKey();
stoFunc.setStoreFuncUDFContextSignature(signature);
String absolutePath = fileNameMap.get(fileNameKey);
if (absolutePath == null) {
absolutePath = stoFunc.relToAbsPathForStoreLocation(
filename,
QueryParserUtils.getCurrentDir(pigContext));
if (absolutePath!=null) {
QueryParserUtils.setHdfsServers(absolutePath, pigContext);
}
fileNameMap.put(fileNameKey, absolutePath);
}
FileSpec fileSpec = new FileSpec(absolutePath, funcSpec);
LOStore op = new LOStore(plan, fileSpec, stoFunc, signature);
return buildOp(loc, op, alias, inputAlias, null);
} catch(Exception ex) {
throw new ParserValidationException(intStream, loc, ex);
}
}
示例13: testSpecCtorClassName
import org.apache.pig.builtin.PigStorage; //导入依赖的package包/类
@Test
public void testSpecCtorClassName() {
String pigStorage = PigStorage.class.getName();
FuncSpec fs = new FuncSpec(pigStorage);
Object o = PigContext.instantiateFuncFromSpec(fs);
assertTrue(o instanceof PigStorage);
}
示例14: testNumericEq
import org.apache.pig.builtin.PigStorage; //导入依赖的package包/类
@Test
public void testNumericEq() throws Throwable {
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
if(i % 5 == 0) {
ps.println(i + ":" + (double)i);
} else {
ps.println(i + ":" + (i-1));
}
}
ps.close();
pig.registerQuery("A=load '"
+ Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ "' using "+PigStorage.class.getName() +"(':');");
String query = "A = filter A by ($0 == $1 and $0 <= $1);";
log.info(query);
pig.registerQuery(query);
Iterator<Tuple> it = pig.openIterator("A");
tmpFile.delete();
while(it.hasNext()) {
Tuple t = it.next();
Double first = Double.valueOf(t.get(0).toString());
Double second = Double.valueOf(t.get(1).toString());
assertEquals(first, second);
String sfirst = t.get(0).toString();
String ssecond = t.get(1).toString();
assertFalse(sfirst.equals(ssecond));
}
}
示例15: testNumericNeq
import org.apache.pig.builtin.PigStorage; //导入依赖的package包/类
@Test
public void testNumericNeq() throws Throwable {
File tmpFile = File.createTempFile("test", "txt");
PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
for(int i = 0; i < LOOP_COUNT; i++) {
if(i % 5 == 0) {
ps.println("1:1");
} else {
ps.println("2:3");
}
}
ps.close();
pig.registerQuery("A=load '"
+ Util.generateURI(tmpFile.toString(), pig.getPigContext())
+ "' using " + PigStorage.class.getName() + "(':');");
String query = "A = filter A by $0 != $1;";
log.info(query);
pig.registerQuery(query);
Iterator<Tuple> it = pig.openIterator("A");
tmpFile.delete();
while(it.hasNext()) {
Tuple t = it.next();
Double first = Double.valueOf(t.get(0).toString());
Double second = Double.valueOf(t.get(1).toString());
assertFalse(first.equals(second));
}
}