Java TimelineClientImpl类代码示例

import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl; //导入依赖的package包/类
@Test
public void testTimelineClientInDSAppMaster() throws Exception {
  ApplicationMaster appMaster = new ApplicationMaster();
  appMaster.appSubmitterUgi =
      UserGroupInformation.createUserForTesting("foo", new String[]{"bar"});
  Configuration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
  appMaster.startTimelineClient(conf);
  Assert.assertEquals(appMaster.appSubmitterUgi,
      ((TimelineClientImpl)appMaster.timelineClient).getUgi());
}
 

import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl; //导入依赖的package包/类
public void map(IntWritable key, IntWritable val, Context context) throws IOException {
  TimelineClient tlc = new TimelineClientImpl();
  Configuration conf = context.getConfiguration();

  final int kbs = conf.getInt(KBS_SENT, KBS_SENT_DEFAULT);

  long totalTime = 0;
  final int testtimes = conf.getInt(TEST_TIMES, TEST_TIMES_DEFAULT);
  final Random rand = new Random();
  final TaskAttemptID taskAttemptId = context.getTaskAttemptID();
  final char[] payLoad = new char[kbs * 1024];

  for (int i = 0; i < testtimes; i++) {
    // Generate a fixed length random payload
    for (int xx = 0; xx < kbs * 1024; xx++) {
      int alphaNumIdx =
          rand.nextInt(ALPHA_NUMS.length);
      payLoad[xx] = ALPHA_NUMS[alphaNumIdx];
    }
    String entId = taskAttemptId + "_" + Integer.toString(i);
    final TimelineEntity entity = new TimelineEntity();
    entity.setEntityId(entId);
    entity.setEntityType("FOO_ATTEMPT");
    entity.addOtherInfo("PERF_TEST", payLoad);
    // add an event
    TimelineEvent event = new TimelineEvent();
    event.setTimestamp(System.currentTimeMillis());
    event.setEventType("foo_event");
    entity.addEvent(event);

    // use the current user for this purpose
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    long startWrite = System.nanoTime();
    try {
      tlc.putEntities(entity);
    } catch (Exception e) {
      context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_FAILURES).
          increment(1);
      LOG.error("writing to the timeline service failed", e);
    }
    long endWrite = System.nanoTime();
    totalTime += TimeUnit.NANOSECONDS.toMillis(endWrite-startWrite);
  }
  LOG.info("wrote " + testtimes + " entities (" + kbs*testtimes +
      " kB) in " + totalTime + " ms");
  context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).
      increment(totalTime);
  context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).
      increment(testtimes);
  context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_KBS).
      increment(kbs*testtimes);
}
 

import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl; //导入依赖的package包/类
public void map(IntWritable key, IntWritable val, Context context) throws IOException {
  // collect the apps it needs to process
  TimelineClient tlc = new TimelineClientImpl();
  TimelineEntityConverterV1 converter = new TimelineEntityConverterV1();
  JobHistoryFileReplayHelper helper = new JobHistoryFileReplayHelper(context);
  int replayMode = helper.getReplayMode();
  Collection<JobFiles> jobs =
      helper.getJobFiles();
  JobHistoryFileParser parser = helper.getParser();

  if (jobs.isEmpty()) {
    LOG.info(context.getTaskAttemptID().getTaskID() +
        " will process no jobs");
  } else {
    LOG.info(context.getTaskAttemptID().getTaskID() + " will process " +
        jobs.size() + " jobs");
  }
  for (JobFiles job: jobs) {
    // process each job
    String jobIdStr = job.getJobId();
    LOG.info("processing " + jobIdStr + "...");
    JobId jobId = TypeConverter.toYarn(JobID.forName(jobIdStr));
    ApplicationId appId = jobId.getAppId();

    try {
      // parse the job info and configuration
      Path historyFilePath = job.getJobHistoryFilePath();
      Path confFilePath = job.getJobConfFilePath();
      if ((historyFilePath == null) || (confFilePath == null)) {
        continue;
      }
      JobInfo jobInfo =
          parser.parseHistoryFile(historyFilePath);
      Configuration jobConf =
          parser.parseConfiguration(confFilePath);
      LOG.info("parsed the job history file and the configuration file for job "
          + jobIdStr);

      // create entities from job history and write them
      long totalTime = 0;
      Set<TimelineEntity> entitySet =
          converter.createTimelineEntities(jobInfo, jobConf);
      LOG.info("converted them into timeline entities for job " + jobIdStr);
      // use the current user for this purpose
      UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
      long startWrite = System.nanoTime();
      try {
        switch (replayMode) {
        case JobHistoryFileReplayHelper.WRITE_ALL_AT_ONCE:
          writeAllEntities(tlc, entitySet, ugi);
          break;
        case JobHistoryFileReplayHelper.WRITE_PER_ENTITY:
          writePerEntity(tlc, entitySet, ugi);
          break;
        default:
          break;
        }
      } catch (Exception e) {
        context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_FAILURES).
            increment(1);
        LOG.error("writing to the timeline service failed", e);
      }
      long endWrite = System.nanoTime();
      totalTime += TimeUnit.NANOSECONDS.toMillis(endWrite-startWrite);
      int numEntities = entitySet.size();
      LOG.info("wrote " + numEntities + " entities in " + totalTime + " ms");

      context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).
          increment(totalTime);
      context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).
          increment(numEntities);
    } finally {
      context.progress(); // move it along
    }
  }
}
 

import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl; //导入依赖的package包/类
@Public
public static TimelineClient createTimelineClient() {
  TimelineClient client = new TimelineClientImpl();
  return client;
}
 

import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl; //导入依赖的package包/类
/**
 * Create a timeline client. The current UGI when the user initialize the
 * client will be used to do the put and the delegation token operations. The
 * current user may use {@link UserGroupInformation#doAs} another user to
 * construct and initialize a timeline client if the following operations are
 * supposed to be conducted by that user.
 *
 * @return a timeline client
 */
@Public
public static TimelineClient createTimelineClient() {
  TimelineClient client = new TimelineClientImpl();
  return client;
}