当前位置: 首页>>代码示例>>Java>>正文

Java TaskType.REDUCE属性代码示例

本文整理汇总了Java中org.apache.hadoop.mapreduce.v2.api.records.TaskType.REDUCE属性的典型用法代码示例。如果您正苦于以下问题：Java TaskType.REDUCE属性的具体用法？Java TaskType.REDUCE怎么用？Java TaskType.REDUCE使用的例子？那么恭喜您, 这里精选的属性代码示例或许可以为您提供帮助。您也可以进一步了解该属性所在类org.apache.hadoop.mapreduce.v2.api.records.TaskType的用法示例。


在下文中一共展示了TaskType.REDUCE属性的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: dataStatisticsForTask

protected DataStatistics dataStatisticsForTask(TaskId taskID) {
  JobId jobID = taskID.getJobId();
  Job job = context.getJob(jobID);

  if (job == null) {
    return null;
  }

  Task task = job.getTask(taskID);

  if (task == null) {
    return null;
  }

  return task.getType() == TaskType.MAP
          ? mapperStatistics.get(job)
          : task.getType() == TaskType.REDUCE
              ? reducerStatistics.get(job)
              : null;
}
开发者ID:naver，项目名称:hadoop，代码行数:20，代码来源:StartEndTimesBase.java

示例2: getJobTaskAttempts

@GET
@Path("/jobs/{jobid}/tasks/{taskid}/attempts")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TaskAttemptsInfo getJobTaskAttempts(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @PathParam("taskid") String tid) {

  init();
  TaskAttemptsInfo attempts = new TaskAttemptsInfo();
  Job job = getJobFromJobIdString(jid, appCtx);
  checkAccess(job, hsr);
  Task task = getTaskFromTaskIdString(tid, job);

  for (TaskAttempt ta : task.getAttempts().values()) {
    if (ta != null) {
      if (task.getType() == TaskType.REDUCE) {
        attempts.add(new ReduceTaskAttemptInfo(ta, task.getType()));
      } else {
        attempts.add(new TaskAttemptInfo(ta, task.getType(), true));
      }
    }
  }
  return attempts;
}
开发者ID:naver，项目名称:hadoop，代码行数:23，代码来源:AMWebServices.java

示例3: getJobTaskAttemptId

@GET
@Path("/jobs/{jobid}/tasks/{taskid}/attempts/{attemptid}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TaskAttemptInfo getJobTaskAttemptId(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @PathParam("taskid") String tid,
    @PathParam("attemptid") String attId) {

  init();
  Job job = getJobFromJobIdString(jid, appCtx);
  checkAccess(job, hsr);
  Task task = getTaskFromTaskIdString(tid, job);
  TaskAttempt ta = getTaskAttemptFromTaskAttemptString(attId, task);
  if (task.getType() == TaskType.REDUCE) {
    return new ReduceTaskAttemptInfo(ta, task.getType());
  } else {
    return new TaskAttemptInfo(ta, task.getType(), true);
  }
}
开发者ID:naver，项目名称:hadoop，代码行数:18，代码来源:AMWebServices.java

示例4: verifyHsTaskAttempt

public void verifyHsTaskAttempt(JSONObject info, TaskAttempt att,
    TaskType ttype) throws JSONException {
  if (ttype == TaskType.REDUCE) {
    assertEquals("incorrect number of elements", 17, info.length());
  } else {
    assertEquals("incorrect number of elements", 12, info.length());
  }

  verifyTaskAttemptGeneric(att, ttype, info.getString("id"),
      info.getString("state"), info.getString("type"),
      info.getString("rack"), info.getString("nodeHttpAddress"),
      info.getString("diagnostics"), info.getString("assignedContainerId"),
      info.getLong("startTime"), info.getLong("finishTime"),
      info.getLong("elapsedTime"), (float) info.getDouble("progress"));

  if (ttype == TaskType.REDUCE) {
    verifyReduceTaskAttemptGeneric(att, info.getLong("shuffleFinishTime"),
        info.getLong("mergeFinishTime"), info.getLong("elapsedShuffleTime"),
        info.getLong("elapsedMergeTime"), info.getLong("elapsedReduceTime"));
  }
}
开发者ID:naver，项目名称:hadoop，代码行数:21，代码来源:TestHsWebServicesAttempts.java

示例5: addTask

protected void addTask(Task task) {
  synchronized (tasksSyncHandle) {
    if (lazyTasksCopyNeeded) {
      Map<TaskId, Task> newTasks = new LinkedHashMap<TaskId, Task>();
      newTasks.putAll(tasks);
      tasks = newTasks;
      lazyTasksCopyNeeded = false;
    }
  }
  tasks.put(task.getID(), task);
  if (task.getType() == TaskType.MAP) {
    mapTasks.add(task.getID());
  } else if (task.getType() == TaskType.REDUCE) {
    reduceTasks.add(task.getID());
  }
  metrics.waitingTask(task);
}
开发者ID:naver，项目名称:hadoop，代码行数:17，代码来源:JobImpl.java

示例6: getJobTaskAttempts

@GET
@Path("/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TaskAttemptsInfo getJobTaskAttempts(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @PathParam("taskid") String tid) {

  init();
  TaskAttemptsInfo attempts = new TaskAttemptsInfo();
  Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
  checkAccess(job, hsr);
  Task task = AMWebServices.getTaskFromTaskIdString(tid, job);
  for (TaskAttempt ta : task.getAttempts().values()) {
    if (ta != null) {
      if (task.getType() == TaskType.REDUCE) {
        attempts.add(new ReduceTaskAttemptInfo(ta, task.getType()));
      } else {
        attempts.add(new TaskAttemptInfo(ta, task.getType(), false));
      }
    }
  }
  return attempts;
}
开发者ID:naver，项目名称:hadoop，代码行数:22，代码来源:HsWebServices.java

示例7: verifyAMTaskAttemptXML

public void verifyAMTaskAttemptXML(Element element, TaskAttempt att,
    TaskType ttype) {
  verifyTaskAttemptGeneric(att, ttype,
      WebServicesTestUtils.getXmlString(element, "id"),
      WebServicesTestUtils.getXmlString(element, "state"),
      WebServicesTestUtils.getXmlString(element, "type"),
      WebServicesTestUtils.getXmlString(element, "rack"),
      WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
      WebServicesTestUtils.getXmlString(element, "diagnostics"),
      WebServicesTestUtils.getXmlString(element, "assignedContainerId"),
      WebServicesTestUtils.getXmlLong(element, "startTime"),
      WebServicesTestUtils.getXmlLong(element, "finishTime"),
      WebServicesTestUtils.getXmlLong(element, "elapsedTime"),
      WebServicesTestUtils.getXmlFloat(element, "progress"));

  if (ttype == TaskType.REDUCE) {
    verifyReduceTaskAttemptGeneric(att,
        WebServicesTestUtils.getXmlLong(element, "shuffleFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "mergeFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedShuffleTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedMergeTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedReduceTime"));
  }
}
开发者ID:naver，项目名称:hadoop，代码行数:24，代码来源:TestAMWebServicesAttempts.java

示例8: verifyAMTaskAttempt

public void verifyAMTaskAttempt(JSONObject info, TaskAttempt att,
    TaskType ttype) throws JSONException {
  if (ttype == TaskType.REDUCE) {
    assertEquals("incorrect number of elements", 17, info.length());
  } else {
    assertEquals("incorrect number of elements", 12, info.length());
  }

  verifyTaskAttemptGeneric(att, ttype, info.getString("id"),
      info.getString("state"), info.getString("type"),
      info.getString("rack"), info.getString("nodeHttpAddress"),
      info.getString("diagnostics"), info.getString("assignedContainerId"),
      info.getLong("startTime"), info.getLong("finishTime"),
      info.getLong("elapsedTime"), (float) info.getDouble("progress"));

  if (ttype == TaskType.REDUCE) {
    verifyReduceTaskAttemptGeneric(att, info.getLong("shuffleFinishTime"),
        info.getLong("mergeFinishTime"), info.getLong("elapsedShuffleTime"),
        info.getLong("elapsedMergeTime"), info.getLong("elapsedReduceTime"));
  }
}
开发者ID:naver，项目名称:hadoop，代码行数:21，代码来源:TestAMWebServicesAttempts.java

示例9: verifyHsTaskAttemptXML

public void verifyHsTaskAttemptXML(Element element, TaskAttempt att,
    TaskType ttype) {
  verifyTaskAttemptGeneric(att, ttype,
      WebServicesTestUtils.getXmlString(element, "id"),
      WebServicesTestUtils.getXmlString(element, "state"),
      WebServicesTestUtils.getXmlString(element, "type"),
      WebServicesTestUtils.getXmlString(element, "rack"),
      WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
      WebServicesTestUtils.getXmlString(element, "diagnostics"),
      WebServicesTestUtils.getXmlString(element, "assignedContainerId"),
      WebServicesTestUtils.getXmlLong(element, "startTime"),
      WebServicesTestUtils.getXmlLong(element, "finishTime"),
      WebServicesTestUtils.getXmlLong(element, "elapsedTime"),
      WebServicesTestUtils.getXmlFloat(element, "progress"));

  if (ttype == TaskType.REDUCE) {
    verifyReduceTaskAttemptGeneric(att,
        WebServicesTestUtils.getXmlLong(element, "shuffleFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "mergeFinishTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedShuffleTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedMergeTime"),
        WebServicesTestUtils.getXmlLong(element, "elapsedReduceTime"));
  }
}
开发者ID:naver，项目名称:hadoop，代码行数:24，代码来源:TestHsWebServicesAttempts.java

示例10: getJobTaskAttemptId

@GET
@Path("/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts/{attemptid}")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public TaskAttemptInfo getJobTaskAttemptId(@Context HttpServletRequest hsr,
    @PathParam("jobid") String jid, @PathParam("taskid") String tid,
    @PathParam("attemptid") String attId) {

  init();
  Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
  checkAccess(job, hsr);
  Task task = AMWebServices.getTaskFromTaskIdString(tid, job);
  TaskAttempt ta = AMWebServices.getTaskAttemptFromTaskAttemptString(attId,
      task);
  if (task.getType() == TaskType.REDUCE) {
    return new ReduceTaskAttemptInfo(ta, task.getType());
  } else {
    return new TaskAttemptInfo(ta, task.getType(), false);
  }
}
开发者ID:naver，项目名称:hadoop，代码行数:19，代码来源:HsWebServices.java

示例11: getMemoryRequired

private int getMemoryRequired(Configuration conf, TaskType taskType) {
  int memory = 1024;
  if (taskType == TaskType.MAP)  {
    memory =
        conf.getInt(MRJobConfig.MAP_MEMORY_MB,
            MRJobConfig.DEFAULT_MAP_MEMORY_MB);
  } else if (taskType == TaskType.REDUCE) {
    memory =
        conf.getInt(MRJobConfig.REDUCE_MEMORY_MB,
            MRJobConfig.DEFAULT_REDUCE_MEMORY_MB);
  }
  
  return memory;
}
开发者ID:naver，项目名称:hadoop，代码行数:14，代码来源:TaskAttemptImpl.java

示例12: getCpuRequired

private int getCpuRequired(Configuration conf, TaskType taskType) {
  int vcores = 1;
  if (taskType == TaskType.MAP)  {
    vcores =
        conf.getInt(MRJobConfig.MAP_CPU_VCORES,
            MRJobConfig.DEFAULT_MAP_CPU_VCORES);
  } else if (taskType == TaskType.REDUCE) {
    vcores =
        conf.getInt(MRJobConfig.REDUCE_CPU_VCORES,
            MRJobConfig.DEFAULT_REDUCE_CPU_VCORES);
  }
  
  return vcores;
}
开发者ID:naver，项目名称:hadoop，代码行数:14，代码来源:TaskAttemptImpl.java

示例13: getGpuRequired

private int getGpuRequired(Configuration conf, TaskType taskType) {
  int gcores = 0;
  if (taskType == TaskType.MAP)  {
    gcores =
        conf.getInt(MRJobConfig.MAP_GPU_CORES,
            MRJobConfig.DEFAULT_MAP_GPU_CORES);
  } else if (taskType == TaskType.REDUCE) {
    gcores =
        conf.getInt(MRJobConfig.REDUCE_GPU_CORES,
            MRJobConfig.DEFAULT_REDUCE_GPU_CORES);
  }

  return gcores;
}
开发者ID:naver，项目名称:hadoop，代码行数:14，代码来源:TaskAttemptImpl.java

示例14: transition

@SuppressWarnings("unchecked")
@Override
public TaskAttemptStateInternal transition(TaskAttemptImpl taskAttempt, 
    TaskAttemptEvent event) {
  if(taskAttempt.getID().getTaskId().getTaskType() == TaskType.REDUCE) {
    // after a reduce task has succeeded, its outputs are in safe in HDFS.
    // logically such a task should not be killed. we only come here when
    // there is a race condition in the event queue. E.g. some logic sends
    // a kill request to this attempt when the successful completion event
    // for this task is already in the event queue. so the kill event will
    // get executed immediately after the attempt is marked successful and 
    // result in this transition being exercised.
    // ignore this for reduce tasks
    LOG.info("Ignoring killed event for successful reduce task attempt" +
              taskAttempt.getID().toString());
    return TaskAttemptStateInternal.SUCCEEDED;
  }
  if(event instanceof TaskAttemptKillEvent) {
    TaskAttemptKillEvent msgEvent = (TaskAttemptKillEvent) event;
    //add to diagnostic
    taskAttempt.addDiagnosticInfo(msgEvent.getMessage());
  }

  // not setting a finish time since it was set on success
  assert (taskAttempt.getFinishTime() != 0);

  assert (taskAttempt.getLaunchTime() != 0);
  taskAttempt.eventHandler
      .handle(createJobCounterUpdateEventTAKilled(taskAttempt, true));
  TaskAttemptUnsuccessfulCompletionEvent tauce = createTaskAttemptUnsuccessfulCompletionEvent(
      taskAttempt, TaskAttemptStateInternal.KILLED);
  taskAttempt.eventHandler.handle(new JobHistoryEvent(taskAttempt.attemptId
      .getTaskId().getJobId(), tauce));
  taskAttempt.eventHandler.handle(new TaskTAttemptEvent(
      taskAttempt.attemptId, TaskEventType.T_ATTEMPT_KILLED));
  return TaskAttemptStateInternal.KILLED;
}
开发者ID:naver，项目名称:hadoop，代码行数:37，代码来源:TaskAttemptImpl.java

示例15: taskFailed

private void taskFailed(JobImpl job, Task task) {
  if (task.getType() == TaskType.MAP) {
    job.failedMapTaskCount++;
  } else if (task.getType() == TaskType.REDUCE) {
    job.failedReduceTaskCount++;
  }
  job.addDiagnostic("Task failed " + task.getID());
  job.metrics.failedTask(task);
}
开发者ID:naver，项目名称:hadoop，代码行数:9，代码来源:JobImpl.java


注：本文中的org.apache.hadoop.mapreduce.v2.api.records.TaskType.REDUCE属性示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。