本文整理汇总了Java中org.apache.tez.mapreduce.hadoop.MRHelpers.addMROutputLegacy方法的典型用法代码示例。如果您正苦于以下问题：Java MRHelpers.addMROutputLegacy方法的具体用法？Java MRHelpers.addMROutputLegacy怎么用？Java MRHelpers.addMROutputLegacy使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.tez.mapreduce.hadoop.MRHelpers的用法示例。
在下文中一共展示了MRHelpers.addMROutputLegacy方法的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: createVertexForStage
import org.apache.tez.mapreduce.hadoop.MRHelpers; //导入方法依赖的package包/类
private Vertex createVertexForStage(Configuration stageConf,
Map<String, LocalResource> jobLocalResources,
List<TaskLocationHint> locations, int stageNum, int totalStages)
throws IOException {
// stageNum starts from 0, goes till numStages - 1
boolean isMap = false;
if (stageNum == 0) {
isMap = true;
}
int numTasks = isMap ? stageConf.getInt(MRJobConfig.NUM_MAPS, 0)
: stageConf.getInt(MRJobConfig.NUM_REDUCES, 0);
String processorName = isMap ? MapProcessor.class.getName()
: ReduceProcessor.class.getName();
String vertexName = null;
if (isMap) {
vertexName = MultiStageMRConfigUtil.getInitialMapVertexName();
} else {
if (stageNum == totalStages - 1) {
vertexName = MultiStageMRConfigUtil.getFinalReduceVertexName();
} else {
vertexName = MultiStageMRConfigUtil
.getIntermediateStageVertexName(stageNum);
}
}
Resource taskResource = isMap ? MRHelpers.getMapResource(stageConf)
: MRHelpers.getReduceResource(stageConf);
stageConf.set(MRJobConfig.MROUTPUT_FILE_NAME_PREFIX, "part");
byte[] vertexUserPayload = MRHelpers.createUserPayloadFromConf(stageConf);
Vertex vertex = new Vertex(vertexName, new ProcessorDescriptor(processorName).
setUserPayload(vertexUserPayload),
numTasks, taskResource);
if (isMap) {
byte[] mapInputPayload = MRHelpers.createMRInputPayload(vertexUserPayload, null);
MRHelpers.addMRInput(vertex, mapInputPayload, null);
}
// Map only jobs.
if (stageNum == totalStages -1) {
MRHelpers.addMROutputLegacy(vertex, vertexUserPayload);
}
Map<String, String> taskEnv = new HashMap<String, String>();
setupMapReduceEnv(stageConf, taskEnv, isMap);
Map<String, LocalResource> taskLocalResources =
new TreeMap<String, LocalResource>();
// PRECOMMIT Remove split localization for reduce tasks if it's being set
// here
taskLocalResources.putAll(jobLocalResources);
String taskJavaOpts = isMap ? MRHelpers.getMapJavaOpts(stageConf)
: MRHelpers.getReduceJavaOpts(stageConf);
vertex.setTaskEnvironment(taskEnv)
.setTaskLocalFiles(taskLocalResources)
.setTaskLocationsHint(locations)
.setTaskLaunchCmdOpts(taskJavaOpts);
if (!isMap) {
vertex.setVertexManagerPlugin(new VertexManagerPluginDescriptor(
ShuffleVertexManager.class.getName()));
}
if (LOG.isDebugEnabled()) {
LOG.debug("Adding vertex to DAG" + ", vertexName="
+ vertex.getName() + ", processor="
+ vertex.getProcessorDescriptor().getClassName() + ", parallelism="
+ vertex.getParallelism() + ", javaOpts=" + vertex.getTaskLaunchCmdOpts()
+ ", resources=" + vertex.getTaskResource()
// TODO Add localResources and Environment
);
}
return vertex;
}