当前位置: 首页>>代码示例>>Java>>正文

Java JobCounterUpdateEvent.addCounterUpdate方法代码示例

本文整理汇总了Java中org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent.addCounterUpdate方法的典型用法代码示例。如果您正苦于以下问题：Java JobCounterUpdateEvent.addCounterUpdate方法的具体用法？Java JobCounterUpdateEvent.addCounterUpdate怎么用？Java JobCounterUpdateEvent.addCounterUpdate使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent的用法示例。


在下文中一共展示了JobCounterUpdateEvent.addCounterUpdate方法的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: sendLaunchedEvents

import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
private void sendLaunchedEvents() {
  JobCounterUpdateEvent jce = new JobCounterUpdateEvent(attemptId.getTaskId()
      .getJobId());
  jce.addCounterUpdate(attemptId.getTaskId().getTaskType() == TaskType.MAP ?
      JobCounter.TOTAL_LAUNCHED_MAPS : JobCounter.TOTAL_LAUNCHED_REDUCES, 1);
  eventHandler.handle(jce);

  LOG.info("TaskAttempt: [" + attemptId
      + "] using containerId: [" + container.getId() + " on NM: ["
      + StringInterner.weakIntern(container.getNodeId().toString()) + "]");
  TaskAttemptStartedEvent tase =
    new TaskAttemptStartedEvent(TypeConverter.fromYarn(attemptId),
        TypeConverter.fromYarn(attemptId.getTaskId().getTaskType()),
        launchTime, trackerName, httpPort, shufflePort, container.getId(),
        locality.toString(), avataar.toString());
  eventHandler.handle(
      new JobHistoryEvent(attemptId.getTaskId().getJobId(), tase));
}
开发者ID:naver，项目名称:hadoop，代码行数:20，代码来源:TaskAttemptImpl.java

示例2: assignToFailedMap

import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
private ContainerRequest assignToFailedMap(Container allocated) {
  //try to assign to earlierFailedMaps if present
  ContainerRequest assigned = null;
  while (assigned == null && earlierFailedMaps.size() > 0
      && canAssignMaps()) {
    TaskAttemptId tId = earlierFailedMaps.removeFirst();      
    if (maps.containsKey(tId)) {
      assigned = maps.remove(tId);
      JobCounterUpdateEvent jce =
        new JobCounterUpdateEvent(assigned.attemptID.getTaskId().getJobId());
      jce.addCounterUpdate(JobCounter.OTHER_LOCAL_MAPS, 1);
      eventHandler.handle(jce);
      LOG.info("Assigned from earlierFailedMaps");
      break;
    }
  }
  return assigned;
}
开发者ID:naver，项目名称:hadoop，代码行数:20，代码来源:RMContainerAllocator.java

示例3: assignToFailedMap

import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
private ContainerRequest assignToFailedMap(Container allocated) {
  //try to assign to earlierFailedMaps if present
  ContainerRequest assigned = null;
  while (assigned == null && earlierFailedMaps.size() > 0) {
    TaskAttemptId tId = earlierFailedMaps.removeFirst();      
    if (maps.containsKey(tId)) {
      assigned = maps.remove(tId);
      JobCounterUpdateEvent jce =
        new JobCounterUpdateEvent(assigned.attemptID.getTaskId().getJobId());
      jce.addCounterUpdate(JobCounter.OTHER_LOCAL_MAPS, 1);
      eventHandler.handle(jce);
      LOG.info("Assigned from earlierFailedMaps");
      break;
    }
  }
  return assigned;
}
开发者ID:Nextzero，项目名称:hadoop-2.6.0-cdh5.4.3，代码行数:19，代码来源:RMContainerAllocator.java

示例4: createJobCounterUpdateEventTAFailed

import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent; //导入方法依赖的package包/类
private static JobCounterUpdateEvent createJobCounterUpdateEventTAFailed(
    TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) {
  TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
  JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
  
  long slotMillisIncrement = computeSlotMillis(taskAttempt);
  
  if (taskType == TaskType.MAP) {
    jce.addCounterUpdate(JobCounter.NUM_FAILED_MAPS, 1);
    if(!taskAlreadyCompleted) {
      // dont double count the elapsed time
      jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, slotMillisIncrement);
    }
  } else {
    jce.addCounterUpdate(JobCounter.NUM_FAILED_REDUCES, 1);
    if(!taskAlreadyCompleted) {
      // dont double count the elapsed time
      jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, slotMillisIncrement);
    }
  }
  return jce;
}
开发者ID:ict-carch，项目名称:hadoop-plus，代码行数:23，代码来源:TaskAttemptImpl.java

示例5: createJobCounterUpdateEventTAKilled

import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent; //导入方法依赖的package包/类
private static JobCounterUpdateEvent createJobCounterUpdateEventTAKilled(
    TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) {
  TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
  JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
  
  long slotMillisIncrement = computeSlotMillis(taskAttempt);
  
  if (taskType == TaskType.MAP) {
    jce.addCounterUpdate(JobCounter.NUM_KILLED_MAPS, 1);
    if(!taskAlreadyCompleted) {
      // dont double count the elapsed time
      jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, slotMillisIncrement);
    }
  } else {
    jce.addCounterUpdate(JobCounter.NUM_KILLED_REDUCES, 1);
    if(!taskAlreadyCompleted) {
      // dont double count the elapsed time
      jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, slotMillisIncrement);
    }
  }
  return jce;
}
开发者ID:ict-carch，项目名称:hadoop-plus，代码行数:23，代码来源:TaskAttemptImpl.java

示例6: sendLaunchedEvents

import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
private void sendLaunchedEvents() {
  JobCounterUpdateEvent jce = new JobCounterUpdateEvent(attemptId.getTaskId()
      .getJobId());
  
  jce.addCounterUpdate(attemptId.getTaskId().getTaskType() == TaskType.MAP ?
      JobCounter.TOTAL_LAUNCHED_MAPS : JobCounter.TOTAL_LAUNCHED_REDUCES, 1);
  eventHandler.handle(jce);

  LOG.info("TaskAttempt: [" + attemptId
      + "] using containerId: [" + container.getId() + " on NM: ["
      + StringInterner.weakIntern(container.getNodeId().toString()) + "]");
  TaskAttemptStartedEvent tase =
    new TaskAttemptStartedEvent(TypeConverter.fromYarn(attemptId),
        TypeConverter.fromYarn(attemptId.getTaskId().getTaskType()),
        launchTime, trackerName, httpPort, shufflePort, container.getId(),
        locality.toString(), avataar.toString());
  eventHandler.handle(
      new JobHistoryEvent(attemptId.getTaskId().getJobId(), tase));
}
开发者ID:yncxcw，项目名称:FlexMap，代码行数:21，代码来源:TaskAttemptImpl.java

示例7: handle

import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
@Override
public void handle(ContainerAllocatorEvent event) {
  if (event.getType() == ContainerAllocator.EventType.CONTAINER_REQ) {
    LOG.info("Processing the event " + event.toString());
    // Assign the same container ID as the AM
    ContainerId cID =
        ContainerId.newContainerId(getContext().getApplicationAttemptId(),
          this.containerId.getContainerId());
    Container container = recordFactory.newRecordInstance(Container.class);
    container.setId(cID);
    NodeId nodeId = NodeId.newInstance(this.nmHost, this.nmPort);
    container.setNodeId(nodeId);
    container.setContainerToken(null);
    container.setNodeHttpAddress(this.nmHost + ":" + this.nmHttpPort);
    // send the container-assigned event to task attempt

    if (event.getAttemptID().getTaskId().getTaskType() == TaskType.MAP) {
      JobCounterUpdateEvent jce =
          new JobCounterUpdateEvent(event.getAttemptID().getTaskId()
              .getJobId());
      // TODO Setting OTHER_LOCAL_MAP for now.
      jce.addCounterUpdate(JobCounter.OTHER_LOCAL_MAPS, 1);
      eventHandler.handle(jce);
    }
    eventHandler.handle(new TaskAttemptContainerAssignedEvent(
        event.getAttemptID(), container, applicationACLs));
  }
}
开发者ID:naver，项目名称:hadoop，代码行数:30，代码来源:LocalContainerAllocator.java

示例8: updateMillisCounters

import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent; //导入方法依赖的package包/类
private static void updateMillisCounters(JobCounterUpdateEvent jce,
    TaskAttemptImpl taskAttempt) {
  TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
  long duration = (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
  int mbRequired =
      taskAttempt.getMemoryRequired(taskAttempt.conf, taskType);
  int vcoresRequired = taskAttempt.getCpuRequired(taskAttempt.conf, taskType);
  int gcoresRequired = taskAttempt.getGpuRequired(taskAttempt.conf, taskType);

  int minSlotMemSize = taskAttempt.conf.getInt(
    YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
    YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);

  int simSlotsRequired =
      minSlotMemSize == 0 ? 0 : (int) Math.ceil((float) mbRequired
          / minSlotMemSize);

  if (taskType == TaskType.MAP) {
    jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, simSlotsRequired * duration);
    jce.addCounterUpdate(JobCounter.MB_MILLIS_MAPS, duration * mbRequired);
    jce.addCounterUpdate(JobCounter.VCORES_MILLIS_MAPS, duration * vcoresRequired);
    jce.addCounterUpdate(JobCounter.GCORES_MILLIS_MAPS, duration * gcoresRequired);
    jce.addCounterUpdate(JobCounter.MILLIS_MAPS, duration);
  } else {
    jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, simSlotsRequired * duration);
    jce.addCounterUpdate(JobCounter.MB_MILLIS_REDUCES, duration * mbRequired);
    jce.addCounterUpdate(JobCounter.VCORES_MILLIS_REDUCES, duration * vcoresRequired);
    jce.addCounterUpdate(JobCounter.GCORES_MILLIS_REDUCES, duration * gcoresRequired);
    jce.addCounterUpdate(JobCounter.MILLIS_REDUCES, duration);
  }
}
开发者ID:naver，项目名称:hadoop，代码行数:32，代码来源:TaskAttemptImpl.java

示例9: createJobCounterUpdateEventTAFailed

import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent; //导入方法依赖的package包/类
private static JobCounterUpdateEvent createJobCounterUpdateEventTAFailed(
    TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) {
  TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
  JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
  
  if (taskType == TaskType.MAP) {
    jce.addCounterUpdate(JobCounter.NUM_FAILED_MAPS, 1);
  } else {
    jce.addCounterUpdate(JobCounter.NUM_FAILED_REDUCES, 1);
  }
  if (!taskAlreadyCompleted) {
    updateMillisCounters(jce, taskAttempt);
  }
  return jce;
}
开发者ID:naver，项目名称:hadoop，代码行数:16，代码来源:TaskAttemptImpl.java

示例10: createJobCounterUpdateEventTAKilled

import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent; //导入方法依赖的package包/类
private static JobCounterUpdateEvent createJobCounterUpdateEventTAKilled(
    TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) {
  TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
  JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId());
  
  if (taskType == TaskType.MAP) {
    jce.addCounterUpdate(JobCounter.NUM_KILLED_MAPS, 1);
  } else {
    jce.addCounterUpdate(JobCounter.NUM_KILLED_REDUCES, 1);
  }
  if (!taskAlreadyCompleted) {
    updateMillisCounters(jce, taskAttempt);
  }
  return jce;
}
开发者ID:naver，项目名称:hadoop，代码行数:16，代码来源:TaskAttemptImpl.java

示例11: updateMillisCounters

import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent; //导入方法依赖的package包/类
private static void updateMillisCounters(JobCounterUpdateEvent jce,
    TaskAttemptImpl taskAttempt) {
  TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
  long duration = (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());
  int mbRequired =
      taskAttempt.getMemoryRequired(taskAttempt.conf, taskType);
  int vcoresRequired = taskAttempt.getCpuRequired(taskAttempt.conf, taskType);

  int minSlotMemSize = taskAttempt.conf.getInt(
    YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
    YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);

  int simSlotsRequired =
      minSlotMemSize == 0 ? 0 : (int) Math.ceil((float) mbRequired
          / minSlotMemSize);

  if (taskType == TaskType.MAP) {
    jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, simSlotsRequired * duration);
    jce.addCounterUpdate(JobCounter.MB_MILLIS_MAPS, duration * mbRequired);
    jce.addCounterUpdate(JobCounter.VCORES_MILLIS_MAPS, duration * vcoresRequired);
    jce.addCounterUpdate(JobCounter.MILLIS_MAPS, duration);
  } else {
    jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, simSlotsRequired * duration);
    jce.addCounterUpdate(JobCounter.MB_MILLIS_REDUCES, duration * mbRequired);
    jce.addCounterUpdate(JobCounter.VCORES_MILLIS_REDUCES, duration * vcoresRequired);
    jce.addCounterUpdate(JobCounter.MILLIS_REDUCES, duration);
  }
}
开发者ID:aliyun-beta，项目名称:aliyun-oss-hadoop-fs，代码行数:29，代码来源:TaskAttemptImpl.java

示例12: killContainer

import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent; //导入方法依赖的package包/类
@SuppressWarnings("unchecked")
private void killContainer(Context ctxt, PreemptionContainer c){
  ContainerId reqCont = c.getId();
  TaskAttemptId reqTask = ctxt.getTaskAttempt(reqCont);
  LOG.info("Evicting " + reqTask);
  dispatcher.handle(new TaskAttemptEvent(reqTask,
      TaskAttemptEventType.TA_KILL));

  // add preemption to counters
  JobCounterUpdateEvent jce = new JobCounterUpdateEvent(reqTask
          .getTaskId().getJobId());
      jce.addCounterUpdate(JobCounter.TASKS_REQ_PREEMPT, 1);
      dispatcher.handle(jce);
}
开发者ID:aliyun-beta，项目名称:aliyun-oss-hadoop-fs，代码行数:15，代码来源:KillAMPreemptionPolicy.java

示例13: updatePreemptionCounters

import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent; //导入方法依赖的package包/类
@SuppressWarnings({ "unchecked" })
private void updatePreemptionCounters(TaskAttemptId yarnAttemptID) {
  if (!countedPreemptions.contains(yarnAttemptID)) {
    countedPreemptions.add(yarnAttemptID);
    JobCounterUpdateEvent jce = new JobCounterUpdateEvent(yarnAttemptID
        .getTaskId().getJobId());
    jce.addCounterUpdate(JobCounter.TASKS_REQ_PREEMPT, 1);
    eventHandler.handle(jce);
  }
}
开发者ID:aliyun-beta，项目名称:aliyun-oss-hadoop-fs，代码行数:11，代码来源:CheckpointAMPreemptionPolicy.java

示例14: createJobCounterUpdateEventTASucceeded

import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent; //导入方法依赖的package包/类
private static JobCounterUpdateEvent createJobCounterUpdateEventTASucceeded(
    TaskAttemptImpl taskAttempt) {
  long slotMillis = computeSlotMillis(taskAttempt);
  TaskId taskId = taskAttempt.attemptId.getTaskId();
  JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskId.getJobId());
  jce.addCounterUpdate(
    taskId.getTaskType() == TaskType.MAP ?
      JobCounter.SLOTS_MILLIS_MAPS : JobCounter.SLOTS_MILLIS_REDUCES,
      slotMillis);
  return jce;
}
开发者ID:ict-carch，项目名称:hadoop-plus，代码行数:12，代码来源:TaskAttemptImpl.java

示例15: updateMillisCounters

import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent; //导入方法依赖的package包/类
private static void updateMillisCounters(JobCounterUpdateEvent jce,
    TaskAttemptImpl taskAttempt) {
  TaskType taskType = taskAttempt.getID().getTaskId().getTaskType();
  long duration = (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime());

  int mbRequired =
      taskAttempt.getMemoryRequired(taskAttempt.conf, taskType);
  int vcoresRequired = taskAttempt.getCpuRequired(taskAttempt.conf, taskType);

  int minSlotMemSize = taskAttempt.conf.getInt(
    YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
    YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);

  int simSlotsRequired =
      minSlotMemSize == 0 ? 0 : (int) Math.ceil((float) mbRequired
          / minSlotMemSize);

  if (taskType == TaskType.MAP) {
    jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, simSlotsRequired * duration);
    jce.addCounterUpdate(JobCounter.MB_MILLIS_MAPS, duration * mbRequired);
    jce.addCounterUpdate(JobCounter.VCORES_MILLIS_MAPS, duration * vcoresRequired);
    jce.addCounterUpdate(JobCounter.MILLIS_MAPS, duration);
  } else {
    jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, simSlotsRequired * duration);
    jce.addCounterUpdate(JobCounter.MB_MILLIS_REDUCES, duration * mbRequired);
    jce.addCounterUpdate(JobCounter.VCORES_MILLIS_REDUCES, duration * vcoresRequired);
    jce.addCounterUpdate(JobCounter.MILLIS_REDUCES, duration);
  }
}
开发者ID:hopshadoop，项目名称:hops，代码行数:30，代码来源:TaskAttemptImpl.java


注：本文中的org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent.addCounterUpdate方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。