Java LoadFunc.setLocation方法代码示例

import org.apache.pig.LoadFunc; //导入方法依赖的package包/类
private void seekInRightStream(Object firstLeftKey) throws IOException{
    rightLoader = (LoadFunc)PigContext.instantiateFuncFromSpec(rightLoaderFuncSpec);
    
    // check if hadoop distributed cache is used
    if (indexFile != null && rightLoader instanceof DefaultIndexableLoader) {
        DefaultIndexableLoader loader = (DefaultIndexableLoader)rightLoader;
        loader.setIndexFile(indexFile);
    }
    
    // Pass signature of the loader to rightLoader
    // make a copy of the conf to use in calls to rightLoader.
    rightLoader.setUDFContextSignature(signature);
    Job job = new Job(new Configuration(PigMapReduce.sJobConfInternal.get()));
    rightLoader.setLocation(rightInputFileName, job);
    ((IndexableLoadFunc)rightLoader).initialize(job.getConfiguration());
    ((IndexableLoadFunc)rightLoader).seekNear(
            firstLeftKey instanceof Tuple ? (Tuple)firstLeftKey : mTupleFactory.newTuple(firstLeftKey));
}
 

import org.apache.pig.LoadFunc; //导入方法依赖的package包/类
private void seekInRightStream(Object firstLeftKey) throws IOException{
    rightLoader = (LoadFunc)PigContext.instantiateFuncFromSpec(rightLoaderFuncSpec);

    // check if hadoop distributed cache is used
    if (indexFile != null && rightLoader instanceof DefaultIndexableLoader) {
        DefaultIndexableLoader loader = (DefaultIndexableLoader)rightLoader;
        loader.setIndexFile(indexFile);
    }
    
    // Pass signature of the loader to rightLoader
    // make a copy of the conf to use in calls to rightLoader.
    rightLoader.setUDFContextSignature(signature);
    Job job = new Job(new Configuration(PigMapReduce.sJobConfInternal.get()));
    rightLoader.setLocation(rightInputFileName, job);
    ((IndexableLoadFunc)rightLoader).initialize(job.getConfiguration());
    ((IndexableLoadFunc)rightLoader).seekNear(
            firstLeftKey instanceof Tuple ? (Tuple)firstLeftKey : mTupleFactory.newTuple(firstLeftKey));
}
 

import org.apache.pig.LoadFunc; //导入方法依赖的package包/类
public static POLoad createPOLoadWithSize(long size, LoadFunc loadFunc) throws Exception {
    File file = File.createTempFile("tempFile", ".tmp");
    file.deleteOnExit();
    RandomAccessFile f = new RandomAccessFile(file, "rw");
    f.setLength(size);
    f.close();

    loadFunc.setLocation(file.getAbsolutePath(), new org.apache.hadoop.mapreduce.Job(CONF));
    FuncSpec funcSpec = new FuncSpec(loadFunc.getClass().getCanonicalName());
    POLoad poLoad = new POLoad(new OperatorKey(), loadFunc);
    poLoad.setLFile(new FileSpec(file.getAbsolutePath(), funcSpec));
    poLoad.setPc(new PigContext());
    poLoad.setUp();

    return poLoad;
}
 

import org.apache.pig.LoadFunc; //导入方法依赖的package包/类
/**
 * stolen from JobControlCompiler
 * TODO: refactor it to share this
 * @param physicalPlan
 * @param poLoad
 * @param configuration
 * @return
 * @throws java.io.IOException
 */
private static Configuration configureLoader(PhysicalPlan physicalPlan,
		POLoad poLoad, Configuration configuration, PigContext pigContext) throws IOException {

	Job job = new Job(configuration);
	LoadFunc loadFunc = poLoad.getLoadFunc();

	loadFunc.setLocation(poLoad.getLFile().getFileName(), job);

	// stolen from JobControlCompiler
	ArrayList<FileSpec> pigInputs = new ArrayList<FileSpec>();
	//Store the inp filespecs
	pigInputs.add(poLoad.getLFile());

	ArrayList<List<OperatorKey>> inpTargets = Lists.newArrayList();
	ArrayList<String> inpSignatures = Lists.newArrayList();
	ArrayList<Long> inpLimits = Lists.newArrayList();
	//Store the target operators for tuples read
	//from this input
	List<PhysicalOperator> loadSuccessors = physicalPlan.getSuccessors(poLoad);
	List<OperatorKey> loadSuccessorsKeys = Lists.newArrayList();
	if(loadSuccessors!=null){
		for (PhysicalOperator loadSuccessor : loadSuccessors) {
			loadSuccessorsKeys.add(loadSuccessor.getOperatorKey());
		}
	}
	inpTargets.add(loadSuccessorsKeys);
	inpSignatures.add(poLoad.getSignature());
	inpLimits.add(poLoad.getLimit());

	configuration.set("pig.inputs", ObjectSerializer.serialize(pigInputs));
	configuration.set("pig.inpTargets", ObjectSerializer.serialize(inpTargets));
	configuration.set("pig.inpSignatures", ObjectSerializer.serialize(inpSignatures));
	configuration.set("pig.inpLimits", ObjectSerializer.serialize(inpLimits));
	configuration.set("pig.pigContext", ObjectSerializer.serialize(pigContext));
	configuration.set("udf.import.list", ObjectSerializer.serialize(PigContext.getPackageImportList()));
	return configuration;
}
 

import org.apache.pig.LoadFunc; //导入方法依赖的package包/类
/**
 * get the corresponding configuration for the input on which the split
 * is based and merge it with the Conf supplied
 * 
 * package level access so that this is not publicly used elsewhere
 * @throws IOException 
 */
static void mergeSplitSpecificConf(LoadFunc loadFunc, PigSplit pigSplit, Configuration originalConf) 
throws IOException {
    // set up conf with entries from input specific conf
    Job job = new Job(originalConf);
    loadFunc.setLocation(getLoadLocation(pigSplit.getInputIndex(), 
            originalConf), job);
    // The above setLocation call could write to the conf within
    // the job - merge that updated conf with original conf
    ConfigurationUtil.mergeConf(originalConf, job.getConfiguration());
    
}
 

import org.apache.pig.LoadFunc; //导入方法依赖的package包/类
public static POLoad createPOLoadWithSize(long size, LoadFunc loadFunc) throws Exception {
    File file = File.createTempFile("tempFile", ".tmp");
    file.deleteOnExit();
    RandomAccessFile f = new RandomAccessFile(file, "rw");
    f.setLength(size);

    loadFunc.setLocation(file.getAbsolutePath(), new org.apache.hadoop.mapreduce.Job(CONF));
    FuncSpec funcSpec = new FuncSpec(loadFunc.getClass().getCanonicalName());
    POLoad poLoad = new POLoad(new OperatorKey(), loadFunc);
    poLoad.setLFile(new FileSpec(file.getAbsolutePath(), funcSpec));
    poLoad.setPc(new PigContext());
    poLoad.setUp();

    return poLoad;
}
 

import org.apache.pig.LoadFunc; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public void initialize(InputSplit inputSplit,
        TaskAttemptContext taskAttemptContext) throws IOException,
        InterruptedException {

    FileSplit fileSplit = (FileSplit) inputSplit;

    path = fileSplit.getPath();
    String fileName = path.toUri().toString();

    // select the correct load function and initialise
    loadFuncHelper = new LoadFuncHelper(
            taskAttemptContext.getConfiguration());

    FuncSpec funcSpec = loadFuncHelper.determineFunction(fileName);

    if (funcSpec == null) {
        throw new IOException("Cannot determine LoadFunc for "
                + fileName);
    }

    selectedLoadFunc = (LoadFunc) PigContext
            .instantiateFuncFromSpec(funcSpec);

    selectedLoadFunc.setUDFContextSignature(udfSignature);
    selectedLoadFunc.setLocation(fileName,
            new Job(taskAttemptContext.getConfiguration(),
                    taskAttemptContext.getJobName()));

    selectedReader = selectedLoadFunc.getInputFormat()
            .createRecordReader(fileSplit, taskAttemptContext);

    selectedReader.initialize(fileSplit, taskAttemptContext);

    LOG.info("Using LoadFunc " + selectedLoadFunc.getClass().getName()
            + " on " + fileName);

}
 

import org.apache.pig.LoadFunc; //导入方法依赖的package包/类
/**
 * stolen from JobControlCompiler TODO: refactor it to share this
 *
 * @param physicalPlan
 * @param poLoad
 * @param jobConf
 * @return
 * @throws java.io.IOException
 */
private static JobConf configureLoader(PhysicalPlan physicalPlan,
        POLoad poLoad, JobConf jobConf) throws IOException {

    Job job = new Job(jobConf);
    LoadFunc loadFunc = poLoad.getLoadFunc();

    loadFunc.setLocation(poLoad.getLFile().getFileName(), job);

    // stolen from JobControlCompiler
    ArrayList<FileSpec> pigInputs = new ArrayList<FileSpec>();
    // Store the inp filespecs
    pigInputs.add(poLoad.getLFile());

    ArrayList<List<OperatorKey>> inpTargets = Lists.newArrayList();
    ArrayList<String> inpSignatures = Lists.newArrayList();
    ArrayList<Long> inpLimits = Lists.newArrayList();
    // Store the target operators for tuples read
    // from this input
    List<PhysicalOperator> loadSuccessors = physicalPlan
            .getSuccessors(poLoad);
    List<OperatorKey> loadSuccessorsKeys = Lists.newArrayList();
    if (loadSuccessors != null) {
        for (PhysicalOperator loadSuccessor : loadSuccessors) {
            loadSuccessorsKeys.add(loadSuccessor.getOperatorKey());
        }
    }
    inpTargets.add(loadSuccessorsKeys);
    inpSignatures.add(poLoad.getSignature());
    inpLimits.add(poLoad.getLimit());

    jobConf.set("pig.inputs", ObjectSerializer.serialize(pigInputs));
    jobConf.set("pig.inpTargets", ObjectSerializer.serialize(inpTargets));
    jobConf.set("pig.inpSignatures",
            ObjectSerializer.serialize(inpSignatures));
    jobConf.set("pig.inpLimits", ObjectSerializer.serialize(inpLimits));

    return jobConf;
}
 

import org.apache.pig.LoadFunc; //导入方法依赖的package包/类
/**
 * get the corresponding configuration for the input on which the split
 * is based and merge it with the Conf supplied
 *
 * package level access so that this is not publicly used elsewhere
 * @throws IOException
 */
static void mergeSplitSpecificConf(LoadFunc loadFunc, PigSplit pigSplit, Configuration originalConf)
        throws IOException {
    // set up conf with entries from input specific conf
    Job job = new Job(originalConf);
    loadFunc.setLocation(getLoadLocation(pigSplit.getInputIndex(),
            originalConf), job);
    // The above setLocation call could write to the conf within
    // the job - merge that updated conf with original conf
    ConfigurationUtil.mergeConf(originalConf, job.getConfiguration());

}