本文整理汇总了Java中org.apache.hadoop.mapred.TaskStatus.Phase类的典型用法代码示例。如果您正苦于以下问题:Java Phase类的具体用法?Java Phase怎么用?Java Phase使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
Phase类属于org.apache.hadoop.mapred.TaskStatus包,在下文中一共展示了Phase类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: TaskInProgress
import org.apache.hadoop.mapred.TaskStatus.Phase; //导入依赖的package包/类
public TaskInProgress(Task task, JobConf conf, TaskLauncher launcher) {
this.task = task;
this.launcher = launcher;
this.lastProgressReport = System.currentTimeMillis();
this.ttConf = conf;
localJobConf = null;
taskStatus = TaskStatus.createTaskStatus(task.isMapTask(), task.getTaskID(),
0.0f,
task.getNumSlotsRequired(),
task.getState(),
diagnosticInfo.toString(),
"initializing",
getName(),
task.isTaskCleanupTask() ?
TaskStatus.Phase.CLEANUP :
task.isMapTask()? TaskStatus.Phase.MAP:
TaskStatus.Phase.SHUFFLE,
task.getCounters());
taskTimeout = (10 * 60 * 1000);
}
示例2: setTaskFailState
import org.apache.hadoop.mapred.TaskStatus.Phase; //导入依赖的package包/类
private void setTaskFailState(boolean wasFailure) {
// go FAILED_UNCLEAN -> FAILED and KILLED_UNCLEAN -> KILLED always
if (taskStatus.getRunState() == TaskStatus.State.FAILED_UNCLEAN) {
taskStatus.setRunState(TaskStatus.State.FAILED);
} else if (taskStatus.getRunState() ==
TaskStatus.State.KILLED_UNCLEAN) {
taskStatus.setRunState(TaskStatus.State.KILLED);
} else if (task.isMapOrReduce() &&
taskStatus.getPhase() != TaskStatus.Phase.CLEANUP) {
if (wasFailure) {
taskStatus.setRunState(TaskStatus.State.FAILED_UNCLEAN);
} else {
taskStatus.setRunState(TaskStatus.State.KILLED_UNCLEAN);
}
} else {
if (wasFailure) {
taskStatus.setRunState(TaskStatus.State.FAILED);
} else {
taskStatus.setRunState(TaskStatus.State.KILLED);
}
}
}
示例3: processingRate
import org.apache.hadoop.mapred.TaskStatus.Phase; //导入依赖的package包/类
public void processingRate(TaskAttemptID taskId, Task.Counter counterName,
long counterValue, float progress, Phase p) {
TaskInProgress tip = jobtracker.taskidToTIPMap.get(taskId);
Counters counters = tip.getCounters();
if(tip.isMapTask()) {
assert p == Phase.MAP : "Map task but phase is " + p;
} else {
assert ((p != Phase.SHUFFLE) &&
(p != Phase.SORT) &&
(p != Phase.REDUCE)) : "Reduce task, but phase is " + p;
}
TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskId,
progress, 1, TaskStatus.State.RUNNING, "", "",
tip.machineWhereTaskRan(taskId), p , counters);
//need to keep the time
TaskStatus oldStatus = tip.getTaskStatus(taskId);
status.setStartTime(oldStatus.getStartTime());
if(!tip.isMapTask()) {
status.setShuffleFinishTime(oldStatus.getShuffleFinishTime());
status.setSortFinishTime(oldStatus.getSortFinishTime());
}
tip.getCounters().findCounter(counterName).setValue(counterValue);
updateTaskStatus(tip, status);
LOG.info(tip.getCounters().toString());
}
示例4: TaskInProgress
import org.apache.hadoop.mapred.TaskStatus.Phase; //导入依赖的package包/类
public TaskInProgress(Task task, JobConf conf, TaskLauncher launcher) {
this.task = task;
this.launcher = launcher;
this.lastProgressReport = System.currentTimeMillis();
this.defaultJobConf = conf;
localJobConf = null;
taskStatus = TaskStatus.createTaskStatus(task.isMapTask(), task.getTaskID(),
0.0f,
task.getState(),
diagnosticInfo.toString(),
"initializing",
getName(),
task.isTaskCleanupTask() ?
TaskStatus.Phase.CLEANUP :
task.isMapTask()? TaskStatus.Phase.MAP:
TaskStatus.Phase.SHUFFLE,
task.getCounters());
taskTimeout = (10 * 60 * 1000);
}
示例5: TaskInProgress
import org.apache.hadoop.mapred.TaskStatus.Phase; //导入依赖的package包/类
public TaskInProgress(Task task, JobConf conf, TaskLauncher launcher) {
this.task = task;
this.launcher = launcher;
this.lastProgressReport = System.currentTimeMillis();
this.defaultJobConf = conf;
localJobConf = null;
taskStatus = TaskStatus.createTaskStatus(task.isMapTask(), task.getTaskID(),
0.0f,
task.getState(),
diagnosticInfo.toString(),
"initializing",
getName(),
task.isTaskCleanupTask() ?
TaskStatus.Phase.CLEANUP :
task.isMapTask()? TaskStatus.Phase.MAP:
TaskStatus.Phase.SHUFFLE,
task.runOnGPU(),
task.getCounters());
taskTimeout = (10 * 60 * 1000);
}
示例6: reducesInShuffle
import org.apache.hadoop.mapred.TaskStatus.Phase; //导入依赖的package包/类
private List <FetchStatus> reducesInShuffle() {
List <FetchStatus> fList = new ArrayList<FetchStatus>();
for (Map.Entry <JobID, RunningJob> item : runningJobs.entrySet()) {
RunningJob rjob = item.getValue();
if (!rjob.localized) {
continue;
}
JobID jobId = item.getKey();
FetchStatus f;
synchronized (rjob) {
f = rjob.getFetchStatus();
for (TaskInProgress tip : rjob.tasks) {
Task task = tip.getTask();
if (!task.isMapTask()) {
if (((ReduceTask)task).getPhase() ==
TaskStatus.Phase.SHUFFLE) {
if (rjob.getFetchStatus() == null) {
//this is a new job; we start fetching its map events
f = new FetchStatus(jobId,
((ReduceTask)task).getNumMaps());
rjob.setFetchStatus(f);
}
f = rjob.getFetchStatus();
fList.add(f);
break; //no need to check any more tasks belonging to this
}
}
}
}
}
//at this point, we have information about for which of
//the running jobs do we need to query the jobtracker for map
//outputs (actually map events).
return fList;
}
示例7: finishTask
import org.apache.hadoop.mapred.TaskStatus.Phase; //导入依赖的package包/类
public void finishTask(TaskAttemptID taskId) {
TaskInProgress tip = jobtracker.taskidToTIPMap.get(taskId);
TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskId,
1.0f, 1, TaskStatus.State.SUCCEEDED, "", "",
tip.machineWhereTaskRan(taskId),
tip.isMapTask() ? Phase.MAP : Phase.REDUCE, new Counters());
updateTaskStatus(tip, status);
}
示例8: makeRunning
import org.apache.hadoop.mapred.TaskStatus.Phase; //导入依赖的package包/类
private void makeRunning(TaskAttemptID taskId, TaskInProgress tip,
String taskTracker) {
addRunningTaskToTIP(tip, taskId, new TaskTrackerStatus(taskTracker,
JobInProgress.convertTrackerNameToHostName(taskTracker)), true);
TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskId,
0.0f, 1, TaskStatus.State.RUNNING, "", "", taskTracker,
tip.isMapTask() ? Phase.MAP : Phase.REDUCE, new Counters());
updateTaskStatus(tip, status);
}
示例9: progressMade
import org.apache.hadoop.mapred.TaskStatus.Phase; //导入依赖的package包/类
public void progressMade(TaskAttemptID taskId, float progress) {
TaskInProgress tip = jobtracker.taskidToTIPMap.get(taskId);
TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskId,
progress, 1, TaskStatus.State.RUNNING, "", "",
tip.machineWhereTaskRan(taskId),
tip.isMapTask() ? Phase.MAP : Phase.REDUCE, new Counters());
updateTaskStatus(tip, status);
}
示例10: getRate
import org.apache.hadoop.mapred.TaskStatus.Phase; //导入依赖的package包/类
public double getRate(Phase p) {
if (p == Phase.MAP) {
return this.mapRate;
} else if (p == Phase.SHUFFLE) {
return this.copyRate;
} else if (p == Phase.SORT) {
return this.sortRate;
} else if (p == Phase.REDUCE) {
return this.reduceRate;
} else {
throw new RuntimeException("Invalid phase " + p);
}
}
示例11: getProcessingRate
import org.apache.hadoop.mapred.TaskStatus.Phase; //导入依赖的package包/类
/**
* Get the processing rate for this task (e.g. bytes/ms in reduce)
*/
public double getProcessingRate(TaskStatus.Phase phase) {
// we don't have processing rate information for the starting and cleaning
// up phase
if (phase != TaskStatus.Phase.MAP &&
phase != TaskStatus.Phase.SHUFFLE &&
phase != TaskStatus.Phase.SORT &&
phase != TaskStatus.Phase.REDUCE) {
return 0;
}
return processingRates.getRate(getProcessingPhase());
}
示例12: canBeSpeculatedUsingProcessingRate
import org.apache.hadoop.mapred.TaskStatus.Phase; //导入依赖的package包/类
/**
* For the map task, using the bytes processed/sec as the processing rate
* For the reduce task, using different rate for different phase:
* copy: using the bytes copied/sec as the processing rate
* sort: using the accumulated progress rate as the processing rate
* reduce: using the the bytes processed/sec as the processing rate
* @param currentTime
* @return
*/
boolean canBeSpeculatedUsingProcessingRate(long currentTime) {
TaskStatus.Phase p = getProcessingPhase();
// check if the task is on one of following four phases
if ((p != TaskStatus.Phase.MAP) &&
(p != TaskStatus.Phase.SHUFFLE) &&
(p != TaskStatus.Phase.SORT) &&
(p != TaskStatus.Phase.REDUCE)) {
return false;
}
DataStatistics taskStats = job.getRunningTaskStatistics(p);
if (LOG.isDebugEnabled()) {
LOG.debug("TaskID: " + this.id + "processing phase is " + p +
" and processing rate for this phase is " +
getProcessingRate(p));
}
// Find if task should be speculated based on standard deviation
// the max difference allowed between the tasks's progress rate
// and the mean progress rate of sibling tasks.
double maxDiff = (taskStats.std() == 0 ?
taskStats.mean()/3 :
job.getSlowTaskThreshold() * taskStats.std());
// if stddev > mean - we are stuck. cap the max difference at a
// more meaningful number.
maxDiff = Math.min(maxDiff, taskStats.mean() * job.getStddevMeanRatioMax());
return (taskStats.mean() - processingRates.getRate(p) > maxDiff);
}
示例13: makeRunning
import org.apache.hadoop.mapred.TaskStatus.Phase; //导入依赖的package包/类
private void makeRunning(TaskAttemptID taskId, TaskInProgress tip,
String taskTracker, long startTime) {
Phase phase = tip.isMapTask() ? Phase.MAP : Phase.REDUCE;
addRunningTaskToTIP(tip, taskId, new TaskTrackerStatus(taskTracker,
JobInProgress.convertTrackerNameToHostName(taskTracker)), true);
TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskId,
0.0f, 1, TaskStatus.State.RUNNING, "", "", taskTracker,
phase, new Counters());
status.setStartTime(startTime);
updateTaskStatus(tip, status);
}
示例14: failTask
import org.apache.hadoop.mapred.TaskStatus.Phase; //导入依赖的package包/类
public void failTask(TaskAttemptID taskId) {
TaskInProgress tip = jobtracker.taskidToTIPMap.get(taskId);
TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskId,
1.0f, 1, TaskStatus.State.FAILED, "", "", tip
.machineWhereTaskRan(taskId), tip.isMapTask() ? Phase.MAP
: Phase.REDUCE, new Counters());
updateTaskStatus(tip, status);
}
示例15: killTask
import org.apache.hadoop.mapred.TaskStatus.Phase; //导入依赖的package包/类
public void killTask(TaskAttemptID taskId) {
TaskInProgress tip = jobtracker.taskidToTIPMap.get(taskId);
TaskStatus status = TaskStatus.createTaskStatus(tip.isMapTask(), taskId,
1.0f, 1, TaskStatus.State.KILLED, "", "", tip
.machineWhereTaskRan(taskId), tip.isMapTask() ? Phase.MAP
: Phase.REDUCE, new Counters());
updateTaskStatus(tip, status);
}