Java Utils.isLocal方法代码示例

import org.apache.pig.impl.util.Utils; //导入方法依赖的package包/类
private static String addSingleFileToDistributedCache(
        PigContext pigContext, Configuration conf, String filename,
        String prefix) throws IOException {

    if (!pigContext.inIllustrator && !FileLocalizer.fileExists(filename, pigContext)) {
        throw new IOException(
                "Internal error: skew join partition file "
                        + filename + " does not exist");
    }

    String symlink = filename;

    // XXX Hadoop currently doesn't support distributed cache in local mode.
    // This line will be removed after the support is added by Hadoop team.
    if (!Utils.isLocal(pigContext, conf)) {
        symlink = prefix + "_"
                + Integer.toString(System.identityHashCode(filename)) + "_"
                + Long.toString(System.currentTimeMillis());
        filename = filename + "#" + symlink;
        setupDistributedCache(pigContext, conf, new String[] { filename },
                false);
    }

    return symlink;
}
 

import org.apache.pig.impl.util.Utils; //导入方法依赖的package包/类
@Override
public void visitMergeJoin(POMergeJoin join) throws VisitorException {

    // XXX Hadoop currently doesn't support distributed cache in local mode.
    // This line will be removed after the support is added
    if (Utils.isLocal(pigContext, conf)) return;

    String indexFile = join.getIndexFile();

    // merge join may not use an index file
    if (indexFile == null) return;

    try {
        String symlink = addSingleFileToDistributedCache(pigContext,
                conf, indexFile, "indexfile_");
        join.setIndexFile(symlink);
    } catch (IOException e) {
        String msg = "Internal error. Distributed cache could not " +
                "be set up for merge join index file";
        throw new VisitorException(msg, e);
    }
}
 

import org.apache.pig.impl.util.Utils; //导入方法依赖的package包/类
@Override
public void visitMergeCoGroup(POMergeCogroup mergeCoGrp)
        throws VisitorException {

    // XXX Hadoop currently doesn't support distributed cache in local mode.
    // This line will be removed after the support is added
    if (Utils.isLocal(pigContext, conf)) return;

    String indexFile = mergeCoGrp.getIndexFileName();

    if (indexFile == null) throw new VisitorException("No index file");

    try {
        String symlink = addSingleFileToDistributedCache(pigContext,
                conf, indexFile, "indexfile_mergecogrp_");
        mergeCoGrp.setIndexFileName(symlink);
    } catch (IOException e) {
        String msg = "Internal error. Distributed cache could not " +
                "be set up for merge cogrp index file";
        throw new VisitorException(msg, e);
    }
}
 

import org.apache.pig.impl.util.Utils; //导入方法依赖的package包/类
@Override
public void visitFRJoin(POFRJoin join) throws VisitorException {

    // XXX Hadoop currently doesn't support distributed cache in local mode.
    // This line will be removed after the support is added
    if (Utils.isLocal(pigContext, conf)) return;

    // set up distributed cache for the replicated files
    FileSpec[] replFiles = join.getReplFiles();
    ArrayList<String> replicatedPath = new ArrayList<String>();

    FileSpec[] newReplFiles = new FileSpec[replFiles.length];
    long maxSize = Long.valueOf(pigContext.getProperties().getProperty(
            PigConfiguration.PIG_JOIN_REPLICATED_MAX_BYTES, "1000000000"));

    // the first input is not replicated
    long sizeOfReplicatedInputs = 0;
    try {
        for (int i = 0; i < replFiles.length; i++) {
            // ignore fragmented file
            String symlink = "";
            if (i != join.getFragment()) {
                symlink = "pigrepl_" + join.getOperatorKey().toString() + "_"
                        + Integer.toString(System.identityHashCode(
                                replFiles[i].getFileName()))
                                + "_" + Long.toString(System.currentTimeMillis())
                                + "_" + i;
                replicatedPath.add(replFiles[i].getFileName() + "#"
                        + symlink);

                Path path = new Path(replFiles[i].getFileName());
                FileSystem fs = path.getFileSystem(conf);
                sizeOfReplicatedInputs +=
                        MapRedUtil.getPathLength(fs, fs.getFileStatus(path), maxSize);
            }
            newReplFiles[i] = new FileSpec(symlink,
                    (replFiles[i] == null ? null : replFiles[i].getFuncSpec()));
        }

        join.setReplFiles(newReplFiles);

        if (sizeOfReplicatedInputs > maxSize) {
            throw new VisitorException("Replicated input files size: "
                    + sizeOfReplicatedInputs + " exceeds " +
                    PigConfiguration.PIG_JOIN_REPLICATED_MAX_BYTES + ": " + maxSize);
        }

        setupDistributedCache(pigContext, conf, replicatedPath
                .toArray(new String[0]), false);
    } catch (IOException e) {
        String msg = "Internal error. Distributed cache could not " +
                "be set up for the replicated files";
        throw new VisitorException(msg, e);
    }
}