Java LoadFunc.getPathStrings方法代码示例

import org.apache.pig.LoadFunc; //导入方法依赖的package包/类
/**
 * Gets the list of paths from the pathString specified which may contain
 * comma-separated paths and glob style path
 *
 * @throws IOException
 */
public static Set<Path> getPaths(String pathString, Configuration conf, boolean failIfNotFound)
        throws IOException {
    Set<Path> paths = new HashSet<Path>();
    String[] pathStrs = LoadFunc.getPathStrings(pathString);
    for (String pathStr : pathStrs) {
        FileSystem fs = FileSystem.get(new Path(pathStr).toUri(), conf);
        FileStatus[] matchedFiles = fs.globStatus(new Path(pathStr), PATH_FILTER);
        if (matchedFiles == null || matchedFiles.length == 0) {
            if (failIfNotFound) {
                throw new IOException("Input Pattern " + pathStr + " matches 0 files");
            } else {
                continue;
            }
        }
        for (FileStatus file : matchedFiles) {
            paths.add(file.getPath());
        }
    }
    return paths;
}
 

import org.apache.pig.LoadFunc; //导入方法依赖的package包/类
/**
 * Adds all non-hidden directories and subdirectories to set param
 * it supports comma-separated input paths and glob style path
 *
 * @throws IOException
 */
public static boolean getAllSubDirs(Path path, Configuration conf,
        Set<Path> paths) throws IOException {
    String[] pathStrs = LoadFunc.getPathStrings(path.toString());
    for (String pathStr : pathStrs) {
        FileSystem fs = FileSystem.get(new Path(pathStr).toUri(), conf);
        FileStatus[] matchedFiles = fs.globStatus(new Path(pathStr), PATH_FILTER);
        if (matchedFiles == null || matchedFiles.length == 0) {
            return false;
        }
        for (FileStatus file : matchedFiles) {
            getAllSubDirsInternal(file, conf, paths, fs);
        }
    }
    return true;
}
 

import org.apache.pig.LoadFunc; //导入方法依赖的package包/类
/**
 * Get the input size for as many inputs as possible. Inputs that do not report
 * their size nor can pig look that up itself are excluded from this size.
 */
static long getTotalInputFileSize(Configuration conf,
                                  List<POLoad> lds, Job job) throws IOException {
    long totalInputFileSize = 0;
    boolean foundSize = false;
    for (POLoad ld : lds) {
        long size = getInputSizeFromLoader(ld, job);
        if (size > -1) { foundSize = true; }
        if (size > 0) {
            totalInputFileSize += size;
            continue;
        }
        // the input file location might be a list of comma separated files,
        // separate them out
        for (String location : LoadFunc.getPathStrings(ld.getLFile().getFileName())) {
            if (UriUtil.isHDFSFileOrLocalOrS3N(location)) {
                Path path = new Path(location);
                FileSystem fs = path.getFileSystem(conf);
                FileStatus[] status = fs.globStatus(path);
                if (status != null) {
                    for (FileStatus s : status) {
                        totalInputFileSize += Utils.getPathLength(fs, s);
                        foundSize = true;
                    }
                }
            }
        }
    }
    return foundSize ? totalInputFileSize : -1;
}
 

import org.apache.pig.LoadFunc; //导入方法依赖的package包/类
private static String getInputFile(String file) {
    String locations[] = LoadFunc.getPathStrings(file);
    if (locations.length == 1)
        return System.getProperty("user.dir") + "/" + basedir
                + file;
    else {
        ArrayList<String> pathStrings = new ArrayList<String>();
        for (int index = 0; index < locations.length; index++) {
            String f = System.getProperty("user.dir") + "/"
                    + basedir + locations[index].trim();
            pathStrings.add(f);
        }
        return LoadFunc.join(pathStrings, ",");
    }
}
 

import org.apache.pig.LoadFunc; //导入方法依赖的package包/类
/**
 * Get the input size for as many inputs as possible. Inputs that do not report
 * their size nor can pig look that up itself are excluded from this size.
 * 
 * @param conf Configuration
 * @param lds List of POLoads
 * @param job Job
 * @param max Maximum value of total input size that will trigger exit. Many
 * times we're only interested whether the total input size is greater than
 * X or not. In such case, we can exit the function early as soon as the max
 * is reached.
 * @return
 * @throws IOException
 */
static long getTotalInputFileSize(Configuration conf,
        List<POLoad> lds, Job job, long max) throws IOException {
    long totalInputFileSize = 0;
    for (POLoad ld : lds) {
        long size = getInputSizeFromLoader(ld, job);
        if (size > -1) {
            totalInputFileSize += size;
            continue;
        } else {

            // the input file location might be a list of comma separated files,
            // separate them out
            for (String location : LoadFunc.getPathStrings(ld.getLFile().getFileName())) {
                if (UriUtil.isHDFSFileOrLocalOrS3N(location, conf)) {
                    Path path = new Path(location);
                    FileSystem fs = path.getFileSystem(conf);
                    FileStatus[] status = fs.globStatus(path);
                    if (status != null) {
                        for (FileStatus s : status) {
                            totalInputFileSize += MapRedUtil.getPathLength(fs, s, max);
                            if (totalInputFileSize > max) {
                                break;
                            }
                        }
                    } else {
                        // If file is not found, we should report -1
                        return -1;
                    }
                } else {
                    // If we cannot estimate size of a location, we should report -1
                    return -1;
                }
            }
        }
    }
    return totalInputFileSize;
}