Java LoadFuncHelper类代码示例

import org.apache.pig.piggybank.storage.allloader.LoadFuncHelper; //导入依赖的package包/类
@Override
public void setLocation(String location, Job job) throws IOException {
    FileInputFormat.setInputPaths(job, location);
    // called on the front end
    conf = job.getConfiguration();
    loadFuncHelper = new LoadFuncHelper(conf);

    if (constructorPassedPartitionFilter != null) {

        pathPartitionerHelper.setPartitionFilterExpression(
                constructorPassedPartitionFilter, AllLoader.class,
                signature);

    }

    getPartitionKeys(location, job);
}
 

import org.apache.pig.piggybank.storage.allloader.LoadFuncHelper; //导入依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public void initialize(InputSplit inputSplit,
        TaskAttemptContext taskAttemptContext) throws IOException,
        InterruptedException {

    FileSplit fileSplit = (FileSplit) inputSplit;

    path = fileSplit.getPath();
    String fileName = path.toUri().toString();

    // select the correct load function and initialise
    loadFuncHelper = new LoadFuncHelper(
            taskAttemptContext.getConfiguration());

    FuncSpec funcSpec = loadFuncHelper.determineFunction(fileName);

    if (funcSpec == null) {
        throw new IOException("Cannot determine LoadFunc for "
                + fileName);
    }

    selectedLoadFunc = (LoadFunc) PigContext
            .instantiateFuncFromSpec(funcSpec);

    selectedLoadFunc.setUDFContextSignature(udfSignature);
    selectedLoadFunc.setLocation(fileName,
            new Job(taskAttemptContext.getConfiguration(),
                    taskAttemptContext.getJobName()));

    selectedReader = selectedLoadFunc.getInputFormat()
            .createRecordReader(fileSplit, taskAttemptContext);

    selectedReader.initialize(fileSplit, taskAttemptContext);

    LOG.info("Using LoadFunc " + selectedLoadFunc.getClass().getName()
            + " on " + fileName);

}
 

import org.apache.pig.piggybank.storage.allloader.LoadFuncHelper; //导入依赖的package包/类
@Before
public void setUp() throws Exception {

    Configuration hadoopConf = new Configuration();
    FileSystem.setDefaultUri(hadoopConf,
            LocalFileSystem.getDefaultUri(hadoopConf));

    if (baseDir == null) {
        configuration = new Properties();
        configuration.setProperty(LoadFuncHelper.FILE_EXTENSION_LOADERS,
                extensionLoaders);

        baseDir = new File("build/test/testAllLoader");
        if (baseDir.exists()) {
            FileUtil.fullyDelete(baseDir);
        }

        assertTrue(baseDir.mkdirs());

        createSimpleDir();
        createDatePartitionDir();
        createLogicPartitionDir();
        createTaggedLogicPartitionDir();
        createFileByContentDir();

        server = new PigServer(ExecType.LOCAL, configuration);

        server.setBatchOn();
    }
}
 

import org.apache.pig.piggybank.storage.allloader.LoadFuncHelper; //导入依赖的package包/类
@Before
public void setUp() throws Exception {

    if (baseDir == null) {

        // we need this here for some strange reason while running ant test
        // the FileSystem.get call in the LoadFuncHelper will try and
        // connect to localhost??
        PigServer pig = new PigServer(ExecType.LOCAL);

        configuration = new Configuration(false);

        configuration.set(LoadFuncHelper.FILE_EXTENSION_LOADERS,
                extensionLoaders);

        helper = new LoadFuncHelper(configuration);

        baseDir = new File("build/test/testLoadFuncHelper");
        if (baseDir.exists()) {
            FileUtil.fullyDelete(baseDir);
        }

        assertTrue(baseDir.mkdirs());

        testFile = new File(baseDir, "testFile.txt");
        FileWriter writer = new FileWriter(testFile);
        try {
            for (int i = 0; i < 100; i++) {
                writer.append("test test\n");
            }
        } finally {
            writer.close();
        }

    }
}
 

import org.apache.pig.piggybank.storage.allloader.LoadFuncHelper; //导入依赖的package包/类
/**
 * Test that we can load files with the correct loaders as per file content.
 * 
 * @throws IOException
 */
@Test
public void testFilesByContentDir() throws IOException {

    server.shutdown();

    configuration.setProperty(LoadFuncHelper.FILE_EXTENSION_LOADERS,
            contentLoaders);

    server = new PigServer(ExecType.LOCAL, configuration);

    server.setBatchOn();
    server.registerFunction(allLoaderName, new FuncSpec(allLoaderFuncSpec));

    server.registerQuery("a = LOAD '" + filesByContentDir.getAbsolutePath()
            + "' using " + allLoaderFuncSpec + " as (p:float, q:float);");

    readRecordsFromLoader(server, "a", fileRecords);

}
 

import org.apache.pig.piggybank.storage.allloader.LoadFuncHelper; //导入依赖的package包/类
/**
 * Test that we can read a tagged logic partitioned directory
 * 
 * @throws IOException
 */
@Test
public void testTaggedLogicPartitionDir() throws IOException {

    server.shutdown();

    configuration.setProperty(LoadFuncHelper.FILE_EXTENSION_LOADERS,
            taggedExtensionLoaders);

    server = new PigServer(ExecType.LOCAL, configuration);

    server.setBatchOn();
    server.registerFunction(allLoaderName, new FuncSpec(allLoaderFuncSpec));

    int i = 0;
    for (String tag : tags) {

        String schema = taggedSchemas[i++];

        server.registerQuery(tag + " = LOAD '"
                + taggedLogicPartitionDir.getAbsolutePath() + "/" + tag
                + "' using " + allLoaderFuncSpec + " as " + schema + ";");

        readRecordsFromLoader(server, tag, fileRecords
                * logicPartitions.length * fileTypes.length);
    }

}