Java LoadFunc类代码示例

import org.apache.pig.LoadFunc; //导入依赖的package包/类
/**
 * Gets the list of paths from the pathString specified which may contain
 * comma-separated paths and glob style path
 *
 * @throws IOException
 */
public static Set<Path> getPaths(String pathString, Configuration conf, boolean failIfNotFound)
        throws IOException {
    Set<Path> paths = new HashSet<Path>();
    String[] pathStrs = LoadFunc.getPathStrings(pathString);
    for (String pathStr : pathStrs) {
        FileSystem fs = FileSystem.get(new Path(pathStr).toUri(), conf);
        FileStatus[] matchedFiles = fs.globStatus(new Path(pathStr), PATH_FILTER);
        if (matchedFiles == null || matchedFiles.length == 0) {
            if (failIfNotFound) {
                throw new IOException("Input Pattern " + pathStr + " matches 0 files");
            } else {
                continue;
            }
        }
        for (FileStatus file : matchedFiles) {
            paths.add(file.getPath());
        }
    }
    return paths;
}
 

import org.apache.pig.LoadFunc; //导入依赖的package包/类
/**
 * Used from the LogicalPlanBuilder
 *
 * @param loader FuncSpec for load function to use for this load.
 * @param schema schema user specified in script, or null if not specified.
 * @param plan logical plan this load is part of.
 * @param conf
 * @param loadFunc the LoadFunc that was instantiated from loader
 * @param signature the signature that will be passed to the LoadFunc
 */
public LOLoad(FileSpec loader, LogicalSchema schema, LogicalPlan plan, Configuration conf, LoadFunc loadFunc, String signature) {
    super("LOLoad", plan);
    this.scriptSchema = schema;
    this.fs = loader;
    this.schemaFile = loader == null ? null : loader.getFileName();
    this.conf = conf;
    this.loadFunc = loadFunc;
    this.signature = signature;
    storeScriptSchema(conf, scriptSchema, signature);
    if (loadFunc != null) {
        this.loadFunc.setUDFContextSignature(signature);
        try {
            this.determinedSchema = getSchemaFromMetaData();
        } catch (FrontendException e) {
            throw new RuntimeException("Can not retrieve schema from loader " + loadFunc, e);
        }
    } else {
        this.determinedSchema = null;
    }
}
 

import org.apache.pig.LoadFunc; //导入依赖的package包/类
private void setAlias(MapReduceOper mro) {
    ArrayList<String> alias = new ArrayList<String>();
String aliasLocationStr = "";
    try {
    ArrayList<String> aliasLocation = new ArrayList<String>();
    new AliasVisitor(mro.mapPlan, alias, aliasLocation).visit();
    aliasLocationStr += "M: "+LoadFunc.join(aliasLocation, ",");
    if (mro.combinePlan != null) {
        aliasLocation = new ArrayList<String>();
        new AliasVisitor(mro.combinePlan, alias, aliasLocation).visit();
        aliasLocationStr += " C: "+LoadFunc.join(aliasLocation, ",");
    }
    aliasLocation = new ArrayList<String>();
    new AliasVisitor(mro.reducePlan, alias, aliasLocation).visit();
    aliasLocationStr += " R: "+LoadFunc.join(aliasLocation, ",");
        if (!alias.isEmpty()) {
            Collections.sort(alias);
        }              
    } catch (VisitorException e) {
        LOG.warn("unable to get alias", e);
    }
aliasMap.put(mro, LoadFunc.join(alias, ","));
aliasLocationMap.put(mro, aliasLocationStr);
}
 

import org.apache.pig.LoadFunc; //导入依赖的package包/类
@Override
protected void processFsCommand(String[] cmdTokens) throws IOException{
    if(mExplain == null) { // process only if not in "explain" mode

        executeBatch();

        int retCode = -1;

        try {
            retCode = shell.run(cmdTokens);
        } catch (Exception e) {
            throw new IOException(e);
        }

        if (retCode != 0 && !mInteractive) {
            String s = LoadFunc.join(
                    (AbstractList<String>) Arrays.asList(cmdTokens), " ");
            throw new IOException("fs command '" + s
                    + "' failed. Please check output logs for details");
        }
    } else {
        log.warn("'fs' statement is ignored while processing 'explain -script' or '-check'");
    }
}
 

import org.apache.pig.LoadFunc; //导入依赖的package包/类
private void initRightLoader(int [] splitsToBeRead) throws IOException{
    PigContext pc = (PigContext) ObjectSerializer
            .deserialize(PigMapReduce.sJobConfInternal.get().get("pig.pigContext"));
    
    Configuration conf = ConfigurationUtil.toConfiguration(pc.getProperties());
    
    // Hadoop security need this property to be set
    if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
        conf.set("mapreduce.job.credentials.binary", 
                System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
    }
    
    //create ReadToEndLoader that will read the given splits in order
    loader = new ReadToEndLoader((LoadFunc)PigContext.instantiateFuncFromSpec(rightLoaderFuncSpec),
            conf, inpLocation, splitsToBeRead);
}
 

import org.apache.pig.LoadFunc; //导入依赖的package包/类
private void instantiateFunc() throws IOException {
    if (caster != null) return;

    if (funcSpec != null) {
        Object obj = PigContext
                .instantiateFuncFromSpec(funcSpec);
        if (obj instanceof LoadFunc) {
            caster = ((LoadFunc)obj).getLoadCaster();
        } else if (obj instanceof StreamToPig) {
            caster = ((StreamToPig)obj).getLoadCaster();
        } else {
            throw new IOException("Invalid class type "
                    + funcSpec.getClassName());
        }
    }
}
 

import org.apache.pig.LoadFunc; //导入依赖的package包/类
/**
 * @param context 
 * 
 */
public PigRecordReader(InputFormat inputformat, PigSplit pigSplit, 
        LoadFunc loadFunc, TaskAttemptContext context, long limit) throws IOException, InterruptedException {
    this.inputformat = inputformat;
    this.pigSplit = pigSplit; 
    this.loadfunc = loadFunc;
    this.context = context;
    this.inputSpecificConf = context.getConfiguration();
    curReader = null;
    progress = 0;
    idx = 0;
    this.limit = limit;
    initNextRecordReader();
    counterGroup = loadFunc.toString();
    doTiming = context.getConfiguration().getBoolean(TIME_UDFS_PROP, false);
}
 

import org.apache.pig.LoadFunc; //导入依赖的package包/类
/**
    * Reads the partition keys from the location i.e the base directory
    * 
    * @param location
    *            String must be the base directory for the partitions
    * @param conf
    * @param loaderClass
    * @throws IOException
    */
   public void setPartitionKeys(String location, Configuration conf,
    Class<? extends LoadFunc> loaderClass, String signature)
    throws IOException {

Set<String> partitionKeys = getPartitionKeys(location, conf);

if (partitionKeys != null) {
    StringBuilder buff = new StringBuilder();
    int i = 0;
    for (String key : partitionKeys) {
	if (i++ != 0) {
	    buff.append(",");
	}

	buff.append(key);
    }

    UDFContext.getUDFContext()
	    .getUDFProperties(loaderClass, new String[] { signature })
	    .setProperty(PARTITION_COLUMNS, buff.toString());
}

   }
 

import org.apache.pig.LoadFunc; //导入依赖的package包/类
/**
 * test {@link TextLoader} - this also tests that {@link TextLoader} is capable
 * of reading data a couple of dirs deep when the input specified is the top
 * level directory
 */
@Test
public void testLFText() throws Exception {
    String input1 = "This is some text.\nWith a newline in it.\n";
    String expected1 = "This is some text.";
    String expected2 = "With a newline in it.";
    Util.createInputFile(cluster,
            "testLFTest-input1.txt",
            new String[] {input1});
    // check that loading the top level dir still reading the file a couple
    // of subdirs below
    LoadFunc text1 = new ReadToEndLoader(new TextLoader(), ConfigurationUtil.
        toConfiguration(cluster.getProperties()), "testLFTest-input1.txt", 0);
    Tuple f1 = text1.getNext();
    Tuple f2 = text1.getNext();
    Util.deleteFile(cluster, "testLFTest-input1.txt");
    assertTrue(expected1.equals(f1.get(0).toString()) &&
        expected2.equals(f2.get(0).toString()));
    Util.createInputFile(cluster, "testLFTest-input2.txt", new String[] {});
    LoadFunc text2 = new ReadToEndLoader(new TextLoader(), ConfigurationUtil.
        toConfiguration(cluster.getProperties()), "testLFTest-input2.txt", 0);
    Tuple f3 = text2.getNext();
    Util.deleteFile(cluster, "testLFTest-input2.txt");
    assertTrue(f3 == null);
}
 

import org.apache.pig.LoadFunc; //导入依赖的package包/类
/**
 * Adds all non-hidden directories and subdirectories to set param
 * it supports comma-separated input paths and glob style path
 *
 * @throws IOException
 */
public static boolean getAllSubDirs(Path path, Configuration conf,
        Set<Path> paths) throws IOException {
    String[] pathStrs = LoadFunc.getPathStrings(path.toString());
    for (String pathStr : pathStrs) {
        FileSystem fs = FileSystem.get(new Path(pathStr).toUri(), conf);
        FileStatus[] matchedFiles = fs.globStatus(new Path(pathStr), PATH_FILTER);
        if (matchedFiles == null || matchedFiles.length == 0) {
            return false;
        }
        for (FileStatus file : matchedFiles) {
            getAllSubDirsInternal(file, conf, paths, fs);
        }
    }
    return true;
}
 

import org.apache.pig.LoadFunc; //导入依赖的package包/类
private void setAlias(MapReduceOper mro) {
    ArrayList<String> alias = new ArrayList<String>();
    String aliasLocationStr = "";
    try {
        ArrayList<String> aliasLocation = new ArrayList<String>();
        new AliasVisitor(mro.mapPlan, alias, aliasLocation).visit();
        aliasLocationStr += "M: "+LoadFunc.join(aliasLocation, ",");
        if (mro.combinePlan != null) {
            aliasLocation = new ArrayList<String>();
            new AliasVisitor(mro.combinePlan, alias, aliasLocation).visit();
            aliasLocationStr += " C: "+LoadFunc.join(aliasLocation, ",");
        }
        aliasLocation = new ArrayList<String>();
        new AliasVisitor(mro.reducePlan, alias, aliasLocation).visit();
        aliasLocationStr += " R: "+LoadFunc.join(aliasLocation, ",");
        if (!alias.isEmpty()) {
            Collections.sort(alias);
        }
    } catch (VisitorException e) {
        LOG.warn("unable to get alias", e);
    }
    aliasMap.put(mro, LoadFunc.join(alias, ","));
    aliasLocationMap.put(mro, aliasLocationStr);
}
 

import org.apache.pig.LoadFunc; //导入依赖的package包/类
@Override
public void visit() throws VisitorException {
    super.visit();
    if (!aliases.isEmpty()) {
        ArrayList<String> aliasList = new ArrayList<String>(aliases);
        ArrayList<String> aliasLocationList = new ArrayList<String>(aliasLocations);
        Collections.sort(aliasList);
        Collections.sort(aliasLocationList);
        alias = LoadFunc.join(aliasList, ",");
        aliasLocation = LoadFunc.join(aliasLocationList, ",");
    }
    StringBuilder sb = new StringBuilder();
    for (int i = featureSet.nextSetBit(0); i >= 0; i = featureSet.nextSetBit(i+1)) {
        if (sb.length() > 0) sb.append(",");
        sb.append(PIG_FEATURE.values()[i].name());
    }
    features = sb.toString();
}
 

import org.apache.pig.LoadFunc; //导入依赖的package包/类
@Override
protected void processFsCommand(String[] cmdTokens) throws IOException {
    filter.validate(PigCommandFilter.Command.FS);
    if(mExplain == null) { // process only if not in "explain" mode

        executeBatch();

        int retCode = -1;

        try {
            retCode = shell.run(cmdTokens);
        } catch (Exception e) {
            throw new IOException(e);
        }

        if (retCode != 0 && !mInteractive) {
            String s = LoadFunc.join(
                    (AbstractList<String>) Arrays.asList(cmdTokens), " ");
            throw new IOException("fs command '" + s
                    + "' failed. Please check output logs for details");
        }
    } else {
        log.warn("'fs' statement is ignored while processing 'explain -script' or '-check'");
    }
}
 

import org.apache.pig.LoadFunc; //导入依赖的package包/类
private void initRightLoader(int [] splitsToBeRead) throws IOException{
    PigContext pc = (PigContext) ObjectSerializer
            .deserialize(PigMapReduce.sJobConfInternal.get().get("pig.pigContext"));
    
    Configuration conf = ConfigurationUtil.toConfiguration(pc.getProperties());
    
    // Hadoop security need this property to be set
    if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
        conf.set(MRConfiguration.JOB_CREDENTIALS_BINARY, 
                System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
    }
    
    //create ReadToEndLoader that will read the given splits in order
    loader = new ReadToEndLoader((LoadFunc)PigContext.instantiateFuncFromSpec(rightLoaderFuncSpec),
            conf, inpLocation, splitsToBeRead);
}
 

import org.apache.pig.LoadFunc; //导入依赖的package包/类
/**
 * @param context
 *
 */
public PigRecordReader(InputFormat<?, ?> inputformat, PigSplit pigSplit,
        LoadFunc loadFunc, TaskAttemptContext context, long limit) throws IOException, InterruptedException {
    this.inputformat = inputformat;
    this.pigSplit = pigSplit;
    this.loadfunc = loadFunc;
    this.context = context;
    this.reporter = PigStatusReporter.getInstance();
    this.inputSpecificConf = context.getConfiguration();
    curReader = null;
    progress = 0;
    idx = 0;
    this.limit = limit;
    initNextRecordReader();
    doTiming = inputSpecificConf.getBoolean(PIG_UDF_PROFILE, false);
    if (doTiming) {
        counterGroup = loadFunc.toString();
        timingFrequency = inputSpecificConf.getLong(PIG_UDF_PROFILE_FREQUENCY, 100L);
    }
}