Java FuncSpec类代码示例

import org.apache.pig.FuncSpec; //导入依赖的package包/类
@SuppressWarnings("unchecked") //still triggers unchecked warning
@Test(expectedExceptions = IllegalArgumentException.class)
public void testRejectionOfNonSimpleType() throws IOException {
  TupleFactory mTupleFactory = TupleFactory.getInstance();
  BagFactory bagFactory = BagFactory.getInstance();
  
  Tuple outerTuple = mTupleFactory.newTuple(1);
  DataBag outerBag = bagFactory.newDefaultBag();
  Tuple innerTuple = mTupleFactory.newTuple(1);
  DataBag innerBag = bagFactory.newDefaultBag();
  innerTuple.set(0, innerBag);
  outerBag.add(innerTuple);
  outerTuple.set(0, outerBag);
  
  String[] ctorArgs = { "128" };
  EvalFunc<Tuple> dataUdf =
      (EvalFunc<Tuple>) PigContext.instantiateFuncFromSpec(new FuncSpec(udfName, ctorArgs));
  dataUdf.exec(outerTuple);
}
 

import org.apache.pig.FuncSpec; //导入依赖的package包/类
@Test
public void check2ValidArg() throws IOException {
  EvalFunc<Tuple> hashUdf =
      (EvalFunc<Tuple>) PigContext.instantiateFuncFromSpec(new FuncSpec(hashUdfName));
  Tuple in, out;
  
  //test String, seed
  in = mTupleFactory.newTuple(2);
  
  in.set(0, new String("1"));
  //2nd is null
  out = hashUdf.exec(in);
  checkOutput(out, false);
  
  in.set(0, new String("1"));
  in.set(1, 9001);
  out = hashUdf.exec(in);
  checkOutput(out, false);
  
  in.set(0, new String("1"));
  in.set(1, 9001L);
  out = hashUdf.exec(in);
  checkOutput(out, false);
}
 

import org.apache.pig.FuncSpec; //导入依赖的package包/类
@Test
public void check3ValidArgs() throws IOException {
  EvalFunc<Tuple> hashUdf =
      (EvalFunc<Tuple>) PigContext.instantiateFuncFromSpec(new FuncSpec(hashUdfName));
  Tuple in, out;
  
  //test multiple integers, seed
  in = mTupleFactory.newTuple(3);
  
  for (int i = 0; i < 10; i++ ) {
    in.set(0, i);
    in.set(1, 9001);
    in.set(2, 7);
    out = hashUdf.exec(in);
    checkOutput(out, true);
  }
}
 

import org.apache.pig.FuncSpec; //导入依赖的package包/类
@Test
public void testLoadEqualityDifferentFuncSpecCtorArgs() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();

    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad load1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp,
            conf);
    lp.add(load1);

    LOLoad load2 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), aschema1, lp,
            conf);
    lp.add(load2);

    assertFalse(load1.isEqual(load2));
}
 

import org.apache.pig.FuncSpec; //导入依赖的package包/类
String buildSortOp(SourceLocation loc, LOSort sort, String alias, String inputAlias, List<LogicalExpressionPlan> plans,
        List<Boolean> ascFlags, FuncSpec fs) throws ParserValidationException {
    sort.setSortColPlans( plans );
    sort.setUserFunc( fs );
    if (ascFlags.isEmpty()) {
        for (int i=0;i<plans.size();i++)
            ascFlags.add(true);
    }
    sort.setAscendingCols( ascFlags );
    alias = buildOp( loc, sort, alias, inputAlias, null );
    try {
        (new ProjectStarExpander(sort.getPlan())).visit(sort);
        (new ProjStarInUdfExpander(sort.getPlan())).visit(sort);
        new SchemaResetter(sort.getPlan(), true).visit(sort);
    } catch (FrontendException e ) {
        throw new ParserValidationException( intStream, loc, e );
    }
    return alias;
}
 

import org.apache.pig.FuncSpec; //导入依赖的package包/类
public void userFuncArity(DataBag input ) throws ExecException {
	String funcSpec = ARITY.class.getName() + "()";
	PORead read = new PORead(new OperatorKey("", r.nextLong()), input);
	List<PhysicalOperator> inputs = new LinkedList<PhysicalOperator>();
	inputs.add(read);
	POUserFunc userFunc = new POUserFunc(new OperatorKey("", r.nextLong()),
			-1, inputs, new FuncSpec(funcSpec));
	Result res = new Result();
	Integer i = null;
	res = userFunc.getNextInteger();
	while (res.returnStatus != POStatus.STATUS_EOP) {
		// System.out.println(res.result);
		int result = (Integer) res.result;
		assertEquals(2, result);
		res = userFunc.getNextInteger();
	}
}
 

import org.apache.pig.FuncSpec; //导入依赖的package包/类
/**
 * Tries to find the schema supported by one of funcSpecs which can
 * be obtained by inserting a set of casts to the input schema
 * @param funcSpecs - mappings provided by udf
 * @param s - input schema
 * @return the funcSpec that supports the schema that is best suited
 *          to s. The best suited schema is one that has the
 *          lowest score as returned by fitPossible().
 */
private FuncSpec bestFitMatch(List<FuncSpec> funcSpecs, Schema s) {
    FuncSpec matchingSpec = null;
    long score = INF;
    long prevBestScore = Long.MAX_VALUE;
    long bestScore = Long.MAX_VALUE;
    for (Iterator<FuncSpec> iterator = funcSpecs.iterator(); iterator.hasNext();) {
        FuncSpec fs = iterator.next();
        score = fitPossible(s,fs.getInputArgsSchema());
        if(score!=INF && score<=bestScore){
            matchingSpec = fs;
            prevBestScore = bestScore;
            bestScore = score;
        }
    }
    if(matchingSpec!=null && bestScore!=prevBestScore)
        return matchingSpec;

    return null;
}
 

import org.apache.pig.FuncSpec; //导入依赖的package包/类
/**
 * Add given uid, loadFuncSpec to mapping
 * @param uid
 * @param loadFuncSpec
 * @throws VisitorException 
 */
private void addUidLoadFuncToMap(long uid, FuncSpec loadFuncSpec)
throws VisitorException{
    if(loadFuncSpec == null){
        return;
    }
    //ensure that uid always matches to same load func
    FuncSpec curFuncSpec = uid2LoadFuncMap.get(uid);
    if(curFuncSpec == null){
        uid2LoadFuncMap.put(uid, loadFuncSpec);
    }else if(! curFuncSpec.equals(loadFuncSpec)){
        String msg = "Bug: uid mapped to two different load functions : " +
        curFuncSpec + " and " + loadFuncSpec;
        throw new VisitorException(msg,2262, PigException.BUG) ;
    }
}
 

import org.apache.pig.FuncSpec; //导入依赖的package包/类
@Test
public void testDatePartitionedFiles() throws IOException {
    int count = 0;

    String funcSpecString = "org.apache.pig.piggybank.storage.HiveColumnarLoader('f1 string,f2 string,f3 string'"
            + ", '" + startingDate + ":" + endingDate + "')";

    System.out.println(funcSpecString);

    PigServer server = new PigServer(ExecType.LOCAL);
    server.setBatchOn();
    server.registerFunction("org.apache.pig.piggybank.storage.HiveColumnarLoader",
            new FuncSpec(funcSpecString));

    server.registerQuery("a = LOAD '" + Util.encodeEscape(datePartitionedDir.getAbsolutePath()) + "' using "
            + funcSpecString + ";");
    Iterator<Tuple> result = server.openIterator("a");

    while ((result.next()) != null) {
        count++;
    }

    Assert.assertEquals(datePartitionedRowCount, count);
}
 

import org.apache.pig.FuncSpec; //导入依赖的package包/类
@Test
public void testLoadEqualityDifferentFunctionNames() throws FrontendException {
    LogicalPlan lp = new LogicalPlan();

    LogicalSchema aschema1 = new LogicalSchema();
    aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
    LOLoad load1 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "y" })), aschema1, lp,
            conf);
    lp.add(load1);

    // Different function names in FuncSpec
    LOLoad load4 = newLOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] { "x", "z" })), aschema1, lp,
            conf);
    lp.add(load4);

    assertFalse(load1.isEqual(load4));
}
 

import org.apache.pig.FuncSpec; //导入依赖的package包/类
@Test
public void test1DayDatePartitionedFiles() throws IOException {
    int count = 0;

    String funcSpecString = "org.apache.pig.piggybank.storage.HiveColumnarLoader('f1 string,f2 string,f3 string'"
            + ", '" + startingDate + ":" + startingDate + "')";

    System.out.println(funcSpecString);

    PigServer server = new PigServer(ExecType.LOCAL);
    server.setBatchOn();
    server.registerFunction("org.apache.pig.piggybank.storage.HiveColumnarLoader",
            new FuncSpec(funcSpecString));

    server.registerQuery("a = LOAD '" + Util.encodeEscape(datePartitionedDir.getAbsolutePath()) + "' using "
            + funcSpecString + ";");
    Iterator<Tuple> result = server.openIterator("a");

    while ((result.next()) != null) {
        count++;
    }

    Assert.assertEquals(50, count);
}
 

import org.apache.pig.FuncSpec; //导入依赖的package包/类
@Test
public void testNumerOfColumnsWhenDatePartitionedFiles() throws IOException {
    int count = 0;

    String funcSpecString = "org.apache.pig.piggybank.storage.HiveColumnarLoader('f1 string,f2 string,f3 string'"
            + ", '" + startingDate + ":" + endingDate + "')";

    System.out.println(funcSpecString);

    PigServer server = new PigServer(ExecType.LOCAL);
    server.setBatchOn();
    server.registerFunction("org.apache.pig.piggybank.storage.HiveColumnarLoader",
            new FuncSpec(funcSpecString));

    server.registerQuery("a = LOAD '" + Util.encodeEscape(datePartitionedDir.getAbsolutePath()) + "' using "
            + funcSpecString + ";");
    Iterator<Tuple> result = server.openIterator("a");
    Tuple t = null;
    while ((t = result.next()) != null) {
        Assert.assertEquals(4, t.size());
        count++;
    }

    Assert.assertEquals(datePartitionedRowCount, count);
}
 

import org.apache.pig.FuncSpec; //导入依赖的package包/类
public static void attachStorePlan(String scope, LogicalPlan lp, String fileName, String func, 
        Operator input, String alias, PigContext pigContext) throws FrontendException {
    func = func == null ? pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName()) : func;

    FuncSpec funcSpec = new FuncSpec( func );
    StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec( funcSpec );
    
    fileName = removeQuotes( fileName );
    FileSpec fileSpec = new FileSpec( fileName, funcSpec );
    String sig = alias + "_" + LogicalPlanBuilder.newOperatorKey(scope);
    stoFunc.setStoreFuncUDFContextSignature(sig);
    LOStore store = new LOStore(lp, fileSpec, stoFunc, sig);
    store.setAlias(alias);

    try {
        stoFunc.relToAbsPathForStoreLocation( fileName, getCurrentDir( pigContext ) );
    } catch (IOException ioe) {
        FrontendException e = new FrontendException(  ioe.getMessage(), ioe );
        throw e;
    }

    lp.add( store );
    lp.connect( input, store );
}
 

import org.apache.pig.FuncSpec; //导入依赖的package包/类
public void poSortUDFWithNull(DataBag input) throws ExecException {
    PORead read = new PORead(new OperatorKey("", r.nextLong()), input);
    List<PhysicalOperator> inputs = new LinkedList<PhysicalOperator>();
    inputs.add(read);
    String funcName = WeirdComparator.class.getName() + "()";
    /*POUserFunc comparator = new POUserFunc(
            new OperatorKey("", r.nextLong()), -1, inputs, funcName);*/
    POUserComparisonFunc comparator = new POUserComparisonFunc(
            new OperatorKey("", r.nextLong()), -1, null, new FuncSpec(funcName));
    POSort sort = new POSort(new OperatorKey("", r.nextLong()), -1, inputs,
            null, null, comparator);
    Tuple t = null;
    Result res1 = sort.getNextTuple();
    // System.out.println(res1.result);
    Result res2 = sort.getNextTuple();
    while (res2.returnStatus != POStatus.STATUS_EOP) {
        int i1 = ((Integer) ((Tuple) res1.result).get(1) == null ? 0 : (Integer) ((Tuple) res1.result).get(1));
        int i2 = ((Integer) ((Tuple) res2.result).get(1) == null ? 0 : (Integer) ((Tuple) res2.result).get(1));
        int i = (i1 - 50) * (i1 - 50) - (i2 - 50) * (i2 - 50);
        assertEquals(true, (i <= 0));
        System.out.println(i + " : " + res2.result);
        res1 = res2;
        res2 = sort.getNextTuple();
    }

}
 

import org.apache.pig.FuncSpec; //导入依赖的package包/类
private void instantiateFunc(FuncSpec fSpec) {
    this.func = (EvalFunc) PigContext.instantiateFuncFromSpec(fSpec);
    this.setSignature(signature);
    this.setFuncInputSchema(signature);
    if (func.getClass().isAnnotationPresent(MonitoredUDF.class)) {
        executor = new MonitoredUDFExecutor(func);
    }
    //the next couple of initializations do not work as intended for the following reasons
    //the reporter and pigLogger are member variables of PhysicalOperator
    //when instanitateFunc is invoked at deserialization time, both
    //reporter and pigLogger are null. They are set during map and reduce calls,
    //making the initializations here basically useless. Look at the processInput
    //method where these variables are re-initialized. At that point, the PhysicalOperator
    //is set up correctly with the reporter and pigLogger references
    this.func.setReporter(getReporter());
    this.func.setPigLogger(pigLogger);
}