本文整理汇总了Java中org.apache.pig.impl.streaming.StreamingCommand.HandleSpec类的典型用法代码示例。如果您正苦于以下问题:Java HandleSpec类的具体用法?Java HandleSpec怎么用?Java HandleSpec使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
HandleSpec类属于org.apache.pig.impl.streaming.StreamingCommand包,在下文中一共展示了HandleSpec类的10个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: visit
import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
@Override
public void visit(LOStream stream) throws FrontendException{
mapToPredLoadFunc(stream);
StreamingCommand command = ((LOStream)stream).getStreamingCommand();
HandleSpec streamOutputSpec = command.getOutputSpec();
FuncSpec streamLoaderSpec = new FuncSpec(streamOutputSpec.getSpec());
setLoadFuncForUids(stream.getSchema(), streamLoaderSpec);
rel2InputFuncMap.put(stream, streamLoaderSpec);
}
示例2: createInputHandler
import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
/**
* Create an <code>InputHandler</code> for the given input specification
* of the <code>StreamingCommand</code>.
*
* @param command <code>StreamingCommand</code>
* @return <code>InputHandler</code> for the given input specification
* @throws ExecException
*/
public static InputHandler createInputHandler(StreamingCommand command)
throws ExecException {
List<HandleSpec> inputSpecs = command.getHandleSpecs(Handle.INPUT);
HandleSpec in = null;
if (inputSpecs == null || (in = inputSpecs.get(0)) == null) {
return new DefaultInputHandler();
}
return (in.name.equals("stdin")) ? new DefaultInputHandler(in) :
new FileInputHandler(in);
}
示例3: createOutputHandler
import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
/**
* Create an <code>OutputHandler</code> for the given output specification
* of the <code>StreamingCommand</code>.
*
* @param command <code>StreamingCommand</code>
* @return <code>OutputHandler</code> for the given output specification
* @throws ExecException
*/
public static OutputHandler createOutputHandler(StreamingCommand command)
throws ExecException {
List<HandleSpec> outputSpecs = command.getHandleSpecs(Handle.OUTPUT);
HandleSpec out = null;
if (outputSpecs == null || (out = outputSpecs.get(0)) == null) {
return new DefaultOutputHandler();
}
return (out.name.equals("stdout")) ? new DefaultOutputHandler(out) :
new FileOutputHandler(out);
}
示例4: FileInputHandler
import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
public FileInputHandler(HandleSpec handleSpec) throws ExecException {
String fileName = handleSpec.name;
serializer =
(PigToStream) PigContext.instantiateFuncFromSpec(handleSpec.spec);
try {
fileOutStream = new FileOutputStream(new File(fileName));
super.bindTo(fileOutStream);
} catch (IOException fnfe) {
int errCode = 2046;
String msg = "Unable to create FileInputHandler.";
throw new ExecException(msg, errCode, PigException.BUG, fnfe);
}
}
示例5: close
import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
public void close() throws IOException {
try {
super.close();
// Copy the secondary outputs of the task to HDFS
Path scriptOutputDir = new Path(this.scriptOutputDir);
FileSystem fs = scriptOutputDir.getFileSystem(job);
List<HandleSpec> outputSpecs = command.getHandleSpecs(Handle.OUTPUT);
if (outputSpecs != null) {
for (int i=1; i < outputSpecs.size(); ++i) {
String fileName = outputSpecs.get(i).getName();
try {
int partition = job.getInt("mapred.task.partition", -1);
fs.copyFromLocalFile(false, true, new Path(fileName),
new Path(
new Path(scriptOutputDir,
fileName),
getOutputName(partition))
);
} catch (IOException ioe) {
int errCode = 6014;
String msg = "Failed to save secondary output '" +
fileName + "' of task: " + taskId;
throw new ExecException(msg, errCode, PigException.REMOTE_ENVIRONMENT, ioe);
}
}
}
} finally {
// Footer for stderr file of the task
writeDebugFooter();
// Close the stderr file on HDFS
if (errorStream != null) {
errorStream.close();
}
}
}
示例6: FileOutputHandler
import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
public FileOutputHandler(HandleSpec handleSpec) throws ExecException {
fileName = handleSpec.name;
deserializer =
(StreamToPig) PigContext.instantiateFuncFromSpec(handleSpec.spec);
}
示例7: DefaultOutputHandler
import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
public DefaultOutputHandler(HandleSpec spec) {
deserializer = (StreamToPig)PigContext.instantiateFuncFromSpec(spec.spec);
}
示例8: DefaultInputHandler
import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
public DefaultInputHandler(HandleSpec spec) {
serializer = (PigToStream)PigContext.instantiateFuncFromSpec(spec.spec);
}
示例9: writeDebugFooter
import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
private void writeDebugFooter() {
processError("===== Task Information Footer =====");
processError("\nEnd time: " + new Date(System.currentTimeMillis()));
processError("\nExit code: " + exitCode);
List<HandleSpec> inputSpecs = command.getHandleSpecs(Handle.INPUT);
HandleSpec inputSpec =
(inputSpecs != null) ? inputSpecs.get(0) : null;
if (inputSpec == null ||
!inputSpec.getSpec().contains("BinaryStorage")) {
processError("\nInput records: " + inputRecords);
}
processError("\nInput bytes: " + inputBytes + " bytes " +
((inputSpec != null) ?
"(" + inputSpec.getName() + " using " +
inputSpec.getSpec() + ")"
: ""));
List<HandleSpec> outputSpecs = command.getHandleSpecs(Handle.OUTPUT);
HandleSpec outputSpec =
(outputSpecs != null) ? outputSpecs.get(0) : null;
if (outputSpec == null ||
!outputSpec.getSpec().contains("BinaryStorage")) {
processError("\nOutput records: " + outputRecords);
}
processError("\nOutput bytes: " + outputBytes + " bytes " +
((outputSpec != null) ?
"(" + outputSpec.getName() + " using " +
outputSpec.getSpec() + ")"
: ""));
if (outputSpecs != null) {
for (int i=1; i < outputSpecs.size(); ++i) {
HandleSpec spec = outputSpecs.get(i);
processError("\n " + new File(spec.getName()).length()
+ " bytes using " + spec.getSpec());
}
}
processError("\n===== * * * =====\n");
}
示例10: transform
import org.apache.pig.impl.streaming.StreamingCommand.HandleSpec; //导入依赖的package包/类
@Override
public void transform(OperatorPlan matched) throws FrontendException {
LogicalRelationalOperator op = (LogicalRelationalOperator)matched.getSources().get(0);
LogicalSchema s = op.getSchema();
// For every field, build a logical plan. If the field has a type
// other than byte array, then the plan will be cast(project). Else
// it will just be project.
LogicalPlan innerPlan = new LogicalPlan();
LOForEach foreach = new LOForEach(currentPlan);
foreach.setInnerPlan(innerPlan);
foreach.setAlias(op.getAlias());
// Insert the foreach into the plan and patch up the plan.
if (currentPlan.getSuccessors(op) == null)
return;
Operator next = currentPlan.getSuccessors(op).get(0);
currentPlan.insertBetween(op, foreach, next);
List<LogicalExpressionPlan> exps = new ArrayList<LogicalExpressionPlan>();
LOGenerate gen = new LOGenerate(innerPlan, exps, new boolean[s.size()]);
innerPlan.add(gen);
// if we are inserting casts in a load and if the loader
// implements determineSchema(), insert casts only where necessary
// Note that in this case, the data coming out of the loader is not
// a BYTEARRAY but is whatever determineSchema() says it is.
LogicalSchema determinedSchema = null;
if(LOLoad.class.getName().equals(getOperatorClassName())) {
determinedSchema = ((LOLoad)op).getDeterminedSchema();
}
else {
determinedSchema = new LogicalSchema();
for (int i=0;i<s.size();i++) {
determinedSchema.addField(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
}
}
for (int i = 0; i < s.size(); i++) {
LogicalSchema.LogicalFieldSchema fs = s.getField(i);
LOInnerLoad innerLoad = new LOInnerLoad(innerPlan, foreach, i);
innerPlan.add(innerLoad);
innerPlan.connect(innerLoad, gen);
LogicalExpressionPlan exp = new LogicalExpressionPlan();
ProjectExpression prj = new ProjectExpression(exp, i, -1, gen);
exp.add(prj);
if (fs.type != DataType.BYTEARRAY && (determinedSchema == null || (!fs.isEqual(determinedSchema.getField(i))))) {
// Either no schema was determined by loader OR the type
// from the "determinedSchema" is different
// from the type specified - so we need to cast
CastExpression cast = new CastExpression(exp, prj, new LogicalSchema.LogicalFieldSchema(fs));
exp.add(cast);
FuncSpec loadFuncSpec = null;
if(op instanceof LOLoad) {
loadFuncSpec = ((LOLoad)op).getFileSpec().getFuncSpec();
} else if (op instanceof LOStream) {
StreamingCommand command = ((LOStream)op).getStreamingCommand();
HandleSpec streamOutputSpec = command.getOutputSpec();
loadFuncSpec = new FuncSpec(streamOutputSpec.getSpec());
} else {
String msg = "TypeCastInserter invoked with an invalid operator class name: " + innerPlan.getClass().getSimpleName();
throw new FrontendException(msg, 2242);
}
cast.setFuncSpec(loadFuncSpec);
}
exps.add(exp);
}
if (op instanceof LOLoad)
((LOLoad)op).setCastInserted(true);
else
((LOStream)op).setCastInserted(true);
}