当前位置: 首页>>代码示例>>Java>>正文


Java HandleSpec类代码示例

本文整理汇总了Java中org.apache.pig.impl.streaming.StreamingCommand.HandleSpec的典型用法代码示例。如果您正苦于以下问题:Java HandleSpec类的具体用法?Java HandleSpec怎么用?Java HandleSpec使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。


HandleSpec类属于org.apache.pig.impl.streaming.StreamingCommand包,在下文中一共展示了HandleSpec类的10个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: visit

import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
@Override
public void visit(LOStream stream) throws FrontendException{
    mapToPredLoadFunc(stream);
    StreamingCommand command = ((LOStream)stream).getStreamingCommand();
    HandleSpec streamOutputSpec = command.getOutputSpec(); 
    FuncSpec streamLoaderSpec = new FuncSpec(streamOutputSpec.getSpec());
    setLoadFuncForUids(stream.getSchema(), streamLoaderSpec);
    rel2InputFuncMap.put(stream, streamLoaderSpec);
}
 
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:10,代码来源:LineageFindRelVisitor.java

示例2: createInputHandler

import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
/**
 * Create an <code>InputHandler</code> for the given input specification
 * of the <code>StreamingCommand</code>.
 * 
 * @param command <code>StreamingCommand</code>
 * @return <code>InputHandler</code> for the given input specification
 * @throws ExecException
 */
public static InputHandler createInputHandler(StreamingCommand command) 
throws ExecException {
    List<HandleSpec> inputSpecs = command.getHandleSpecs(Handle.INPUT);
    
    HandleSpec in = null;
    if (inputSpecs == null || (in = inputSpecs.get(0)) == null) {
        return new DefaultInputHandler();
    }
    
    return (in.name.equals("stdin")) ? new DefaultInputHandler(in) :
                                       new FileInputHandler(in);
}
 
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:21,代码来源:HandlerFactory.java

示例3: createOutputHandler

import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
/**
 * Create an <code>OutputHandler</code> for the given output specification
 * of the <code>StreamingCommand</code>.
 * 
 * @param command <code>StreamingCommand</code>
 * @return <code>OutputHandler</code> for the given output specification
 * @throws ExecException
 */
public static OutputHandler createOutputHandler(StreamingCommand command) 
throws ExecException {
    List<HandleSpec> outputSpecs = command.getHandleSpecs(Handle.OUTPUT);
    
    HandleSpec out = null;
    if (outputSpecs == null || (out = outputSpecs.get(0)) == null) {
        return new DefaultOutputHandler();
    }
    
    return (out.name.equals("stdout")) ? new DefaultOutputHandler(out) :
                                         new FileOutputHandler(out);
}
 
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:21,代码来源:HandlerFactory.java

示例4: FileInputHandler

import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
public FileInputHandler(HandleSpec handleSpec) throws ExecException {
    String fileName = handleSpec.name;
    serializer = 
        (PigToStream) PigContext.instantiateFuncFromSpec(handleSpec.spec);
    
    try {
        fileOutStream = new FileOutputStream(new File(fileName));
        super.bindTo(fileOutStream);
    } catch (IOException fnfe) {
        int errCode = 2046;
        String msg = "Unable to create FileInputHandler.";
        throw new ExecException(msg, errCode, PigException.BUG, fnfe);
    }
}
 
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:15,代码来源:FileInputHandler.java

示例5: close

import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
public void close() throws IOException {
    try {
        super.close();

        // Copy the secondary outputs of the task to HDFS
        Path scriptOutputDir = new Path(this.scriptOutputDir);
        FileSystem fs = scriptOutputDir.getFileSystem(job);
        List<HandleSpec> outputSpecs = command.getHandleSpecs(Handle.OUTPUT);
        if (outputSpecs != null) {
            for (int i=1; i < outputSpecs.size(); ++i) {
                String fileName = outputSpecs.get(i).getName();
                try {
                    int partition = job.getInt("mapred.task.partition", -1);
                    fs.copyFromLocalFile(false, true, new Path(fileName), 
                                         new Path(
                                                 new Path(scriptOutputDir, 
                                                          fileName), 
                                                 getOutputName(partition))
                                        );
                } catch (IOException ioe) {
                    int errCode = 6014; 
                    String msg = "Failed to save secondary output '" + 
                    fileName + "' of task: " + taskId;
                    throw new ExecException(msg, errCode, PigException.REMOTE_ENVIRONMENT, ioe);
                }
            }
    }
    } finally {
        // Footer for stderr file of the task
        writeDebugFooter();
        
        // Close the stderr file on HDFS
        if (errorStream != null) {
            errorStream.close();
        }
    }
}
 
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:38,代码来源:HadoopExecutableManager.java

示例6: FileOutputHandler

import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
public FileOutputHandler(HandleSpec handleSpec) throws ExecException {
    fileName = handleSpec.name;
    deserializer = 
        (StreamToPig) PigContext.instantiateFuncFromSpec(handleSpec.spec);
}
 
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:6,代码来源:FileOutputHandler.java

示例7: DefaultOutputHandler

import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
public DefaultOutputHandler(HandleSpec spec) {
    deserializer = (StreamToPig)PigContext.instantiateFuncFromSpec(spec.spec);
}
 
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:4,代码来源:DefaultOutputHandler.java

示例8: DefaultInputHandler

import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
public DefaultInputHandler(HandleSpec spec) {
    serializer = (PigToStream)PigContext.instantiateFuncFromSpec(spec.spec);
}
 
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:4,代码来源:DefaultInputHandler.java

示例9: writeDebugFooter

import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
private void writeDebugFooter() {
    processError("===== Task Information Footer =====");

    processError("\nEnd time: " + new Date(System.currentTimeMillis()));
    processError("\nExit code: " + exitCode);

    List<HandleSpec> inputSpecs = command.getHandleSpecs(Handle.INPUT);
    HandleSpec inputSpec = 
        (inputSpecs != null) ? inputSpecs.get(0) : null;
    if (inputSpec == null || 
        !inputSpec.getSpec().contains("BinaryStorage")) {
        processError("\nInput records: " + inputRecords);
    }
    processError("\nInput bytes: " + inputBytes + " bytes " +
                ((inputSpec != null) ? 
                        "(" + inputSpec.getName() + " using " + 
                            inputSpec.getSpec() + ")" 
                        : ""));

    List<HandleSpec> outputSpecs = command.getHandleSpecs(Handle.OUTPUT);
    HandleSpec outputSpec = 
        (outputSpecs != null) ? outputSpecs.get(0) : null;
    if (outputSpec == null || 
        !outputSpec.getSpec().contains("BinaryStorage")) {
        processError("\nOutput records: " + outputRecords);
    }
    processError("\nOutput bytes: " + outputBytes + " bytes " +
                 ((outputSpec != null) ? 
                     "(" + outputSpec.getName() + " using " + 
                         outputSpec.getSpec() + ")" 
                     : ""));
    if (outputSpecs != null) {
        for (int i=1; i < outputSpecs.size(); ++i) {
            HandleSpec spec = outputSpecs.get(i);
            processError("\n           " + new File(spec.getName()).length() 
                         + " bytes using " + spec.getSpec());
        }
    }

    processError("\n=====          * * *          =====\n");
}
 
开发者ID:sigmoidanalytics,项目名称:spork-streaming,代码行数:42,代码来源:HadoopExecutableManager.java

示例10: transform

import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
@Override
public void transform(OperatorPlan matched) throws FrontendException {
    LogicalRelationalOperator op = (LogicalRelationalOperator)matched.getSources().get(0);
    LogicalSchema s = op.getSchema();
    // For every field, build a logical plan.  If the field has a type
    // other than byte array, then the plan will be cast(project).  Else
    // it will just be project.
    LogicalPlan innerPlan = new LogicalPlan();
    
    LOForEach foreach = new LOForEach(currentPlan);
    foreach.setInnerPlan(innerPlan);
    foreach.setAlias(op.getAlias());
    
    // Insert the foreach into the plan and patch up the plan.
    if (currentPlan.getSuccessors(op) == null)
        return;
    Operator next = currentPlan.getSuccessors(op).get(0);
    currentPlan.insertBetween(op, foreach, next);
    
    List<LogicalExpressionPlan> exps = new ArrayList<LogicalExpressionPlan>();
    LOGenerate gen = new LOGenerate(innerPlan, exps, new boolean[s.size()]);
    innerPlan.add(gen);
    
    // if we are inserting casts in a load and if the loader
    // implements determineSchema(), insert casts only where necessary
    // Note that in this case, the data coming out of the loader is not
    // a BYTEARRAY but is whatever determineSchema() says it is.
    LogicalSchema determinedSchema = null;
    if(LOLoad.class.getName().equals(getOperatorClassName())) {
        determinedSchema = ((LOLoad)op).getDeterminedSchema();
    }
    else {
        determinedSchema = new LogicalSchema();
        for (int i=0;i<s.size();i++) {
            determinedSchema.addField(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
        }
    }
    for (int i = 0; i < s.size(); i++) {
        LogicalSchema.LogicalFieldSchema fs = s.getField(i);
        
        LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, i);
        innerPlan.add(innerLoad);          
        innerPlan.connect(innerLoad, gen);
        
        LogicalExpressionPlan exp = new LogicalExpressionPlan();
        
        ProjectExpression prj = new ProjectExpression(exp, i, -1, gen);
        exp.add(prj);
        
        if (fs.type != DataType.BYTEARRAY && (determinedSchema == null || (!fs.isEqual(determinedSchema.getField(i))))) {
            // Either no schema was determined by loader OR the type 
            // from the "determinedSchema" is different
            // from the type specified - so we need to cast
            CastExpression cast = new CastExpression(exp, prj, new LogicalSchema.LogicalFieldSchema(fs));
            exp.add(cast);
            FuncSpec loadFuncSpec = null;
            if(op instanceof LOLoad) {
                loadFuncSpec = ((LOLoad)op).getFileSpec().getFuncSpec();
            } else if (op instanceof LOStream) {
                StreamingCommand command = ((LOStream)op).getStreamingCommand();
                HandleSpec streamOutputSpec = command.getOutputSpec(); 
                loadFuncSpec = new FuncSpec(streamOutputSpec.getSpec());
            } else {
                String msg = "TypeCastInserter invoked with an invalid operator class name: " + innerPlan.getClass().getSimpleName();
                throw new FrontendException(msg, 2242);
            }
            cast.setFuncSpec(loadFuncSpec);
        }
        exps.add(exp);
    }
    if (op instanceof LOLoad)
        ((LOLoad)op).setCastInserted(true);
    else
        ((LOStream)op).setCastInserted(true);
}
 
开发者ID:PonIC,项目名称:PonIC,代码行数:76,代码来源:TypeCastInserter.java


注:本文中的org.apache.pig.impl.streaming.StreamingCommand.HandleSpec类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。