Java TimelineEntity.addPrimaryFilter方法代码示例

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; //导入方法依赖的package包/类
private static TimelineEntity generateEntity() {
  TimelineEntity entity = new TimelineEntity();
  entity.setEntityId("entity id");
  entity.setEntityType("entity type");
  entity.setStartTime(System.currentTimeMillis());
  for (int i = 0; i < 2; ++i) {
    TimelineEvent event = new TimelineEvent();
    event.setTimestamp(System.currentTimeMillis());
    event.setEventType("test event type " + i);
    event.addEventInfo("key1", "val1");
    event.addEventInfo("key2", "val2");
    entity.addEvent(event);
  }
  entity.addRelatedEntity("test ref type 1", "test ref id 1");
  entity.addRelatedEntity("test ref type 2", "test ref id 2");
  entity.addPrimaryFilter("pkey1", "pval1");
  entity.addPrimaryFilter("pkey2", "pval2");
  entity.addOtherInfo("okey1", "oval1");
  entity.addOtherInfo("okey2", "oval2");
  entity.setDomainId("domain id 1");
  return entity;
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; //导入方法依赖的package包/类
@Test
public void testYarnACLsNotEnabledForEntity() throws Exception {
  Configuration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, false);
  TimelineACLsManager timelineACLsManager =
      new TimelineACLsManager(conf);
  timelineACLsManager.setTimelineStore(new TestTimelineStore());
  TimelineEntity entity = new TimelineEntity();
  entity.addPrimaryFilter(
      TimelineStore.SystemFilter.ENTITY_OWNER
          .toString(), "owner");
  entity.setDomainId("domain_id_1");
  Assert.assertTrue(
      "Always true when ACLs are not enabled",
      timelineACLsManager.checkAccess(
          UserGroupInformation.createRemoteUser("user"),
          ApplicationAccessType.VIEW_APP, entity));
  Assert.assertTrue(
      "Always true when ACLs are not enabled",
      timelineACLsManager.checkAccess(
          UserGroupInformation.createRemoteUser("user"),
          ApplicationAccessType.MODIFY_APP, entity));
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; //导入方法依赖的package包/类
private static void publishContainerEndEvent(
    final TimelineClient timelineClient, ContainerStatus container,
    String domainId, UserGroupInformation ugi) {
  final TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(container.getContainerId().toString());
  entity.setEntityType(DSEntity.DS_CONTAINER.toString());
  entity.setDomainId(domainId);
  entity.addPrimaryFilter("user", ugi.getShortUserName());
  TimelineEvent event = new TimelineEvent();
  event.setTimestamp(System.currentTimeMillis());
  event.setEventType(DSEvent.DS_CONTAINER_END.toString());
  event.addEventInfo("State", container.getState().name());
  event.addEventInfo("Exit Status", container.getExitStatus());
  entity.addEvent(event);
  try {
    timelineClient.putEntities(entity);
  } catch (YarnException | IOException e) {
    LOG.error("Container end event could not be published for "
        + container.getContainerId().toString(), e);
  }
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; //导入方法依赖的package包/类
private static void publishApplicationAttemptEvent(
    final TimelineClient timelineClient, String appAttemptId,
    DSEvent appEvent, String domainId, UserGroupInformation ugi) {
  final TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(appAttemptId);
  entity.setEntityType(DSEntity.DS_APP_ATTEMPT.toString());
  entity.setDomainId(domainId);
  entity.addPrimaryFilter("user", ugi.getShortUserName());
  TimelineEvent event = new TimelineEvent();
  event.setEventType(appEvent.toString());
  event.setTimestamp(System.currentTimeMillis());
  entity.addEvent(event);
  try {
    timelineClient.putEntities(entity);
  } catch (YarnException | IOException e) {
    LOG.error("App Attempt "
        + (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? "start" : "end")
        + " event could not be published for "
        + appAttemptId.toString(), e);
  }
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; //导入方法依赖的package包/类
private TimelineEntity createJobEntity(JobInfo jobInfo, Configuration conf) {
  TimelineEntity job = new TimelineEntity();
  job.setEntityType(JOB);
  job.setEntityId(jobInfo.getJobId().toString());
  job.setStartTime(jobInfo.getSubmitTime());

  job.addPrimaryFilter("JOBNAME", jobInfo.getJobname());
  job.addPrimaryFilter("USERNAME", jobInfo.getUsername());
  job.addOtherInfo("JOB_QUEUE_NAME", jobInfo.getJobQueueName());
  job.addOtherInfo("SUBMIT_TIME", jobInfo.getSubmitTime());
  job.addOtherInfo("LAUNCH_TIME", jobInfo.getLaunchTime());
  job.addOtherInfo("FINISH_TIME", jobInfo.getFinishTime());
  job.addOtherInfo("JOB_STATUS", jobInfo.getJobStatus());
  job.addOtherInfo("PRIORITY", jobInfo.getPriority());
  job.addOtherInfo("TOTAL_MAPS", jobInfo.getTotalMaps());
  job.addOtherInfo("TOTAL_REDUCES", jobInfo.getTotalReduces());
  job.addOtherInfo("UBERIZED", jobInfo.getUberized());
  job.addOtherInfo("ERROR_INFO", jobInfo.getErrorInfo());

  LOG.info("converted job " + jobInfo.getJobId() + " to a timeline entity");
  return job;
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; //导入方法依赖的package包/类
private static TimelineEntity createAppAttemptEntity(
    ApplicationAttemptId appAttemptId) {
  TimelineEntity entity = new TimelineEntity();
  entity.setEntityType(
      AppAttemptMetricsConstants.ENTITY_TYPE);
  entity.setEntityId(appAttemptId.toString());
  entity.addPrimaryFilter(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER,
      appAttemptId.getApplicationId().toString());
  return entity;
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; //导入方法依赖的package包/类
private static TimelineEntity createContainerEntity(
    ContainerId containerId) {
  TimelineEntity entity = new TimelineEntity();
  entity.setEntityType(
      ContainerMetricsConstants.ENTITY_TYPE);
  entity.setEntityId(containerId.toString());
  entity.addPrimaryFilter(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER,
      containerId.getApplicationAttemptId().toString());
  return entity;
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; //导入方法依赖的package包/类
private static void publishContainerStartEvent(
    final TimelineClient timelineClient, Container container, String domainId,
    UserGroupInformation ugi) {
  final TimelineEntity entity = new TimelineEntity();
  entity.setEntityId(container.getId().toString());
  entity.setEntityType(DSEntity.DS_CONTAINER.toString());
  entity.setDomainId(domainId);
  entity.addPrimaryFilter("user", ugi.getShortUserName());
  TimelineEvent event = new TimelineEvent();
  event.setTimestamp(System.currentTimeMillis());
  event.setEventType(DSEvent.DS_CONTAINER_START.toString());
  event.addEventInfo("Node", container.getNodeId().toString());
  event.addEventInfo("Resources", container.getResource().toString());
  entity.addEvent(event);

  try {
    ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() {
      @Override
      public TimelinePutResponse run() throws Exception {
        return timelineClient.putEntities(entity);
      }
    });
  } catch (Exception e) {
    LOG.error("Container start event could not be published for "
        + container.getId().toString(),
        e instanceof UndeclaredThrowableException ? e.getCause() : e);
  }
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; //导入方法依赖的package包/类
/**
 * Parses the primary filter from the given key at the given offset and adds
 * it to the given entity.
 */
private static void addPrimaryFilter(TimelineEntity entity, byte[] key,
    int offset) throws IOException {
  KeyParser kp = new KeyParser(key, offset);
  String name = kp.getNextString();
  byte[] bytes = kp.getRemainingBytes();
  Object value = fstConf.asObject(bytes);
  entity.addPrimaryFilter(name, value);
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; //导入方法依赖的package包/类
@Test
public void testYarnACLsEnabledForEntity() throws Exception {
  Configuration conf = new YarnConfiguration();
  conf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
  conf.set(YarnConfiguration.YARN_ADMIN_ACL, "admin");
  TimelineACLsManager timelineACLsManager =
      new TimelineACLsManager(conf);
  timelineACLsManager.setTimelineStore(new TestTimelineStore());
  TimelineEntity entity = new TimelineEntity();
  entity.addPrimaryFilter(
      TimelineStore.SystemFilter.ENTITY_OWNER
          .toString(), "owner");
  entity.setDomainId("domain_id_1");
  Assert.assertTrue(
      "Owner should be allowed to view",
      timelineACLsManager.checkAccess(
          UserGroupInformation.createRemoteUser("owner"),
          ApplicationAccessType.VIEW_APP, entity));
  Assert.assertTrue(
      "Reader should be allowed to view",
      timelineACLsManager.checkAccess(
          UserGroupInformation.createRemoteUser("reader"),
          ApplicationAccessType.VIEW_APP, entity));
  Assert.assertFalse(
      "Other shouldn't be allowed to view",
      timelineACLsManager.checkAccess(
          UserGroupInformation.createRemoteUser("other"),
          ApplicationAccessType.VIEW_APP, entity));
  Assert.assertTrue(
      "Admin should be allowed to view",
      timelineACLsManager.checkAccess(
          UserGroupInformation.createRemoteUser("admin"),
          ApplicationAccessType.VIEW_APP, entity));

  Assert.assertTrue(
      "Owner should be allowed to modify",
      timelineACLsManager.checkAccess(
          UserGroupInformation.createRemoteUser("owner"),
          ApplicationAccessType.MODIFY_APP, entity));
  Assert.assertTrue(
      "Writer should be allowed to modify",
      timelineACLsManager.checkAccess(
          UserGroupInformation.createRemoteUser("writer"),
          ApplicationAccessType.MODIFY_APP, entity));
  Assert.assertFalse(
      "Other shouldn't be allowed to modify",
      timelineACLsManager.checkAccess(
          UserGroupInformation.createRemoteUser("other"),
          ApplicationAccessType.MODIFY_APP, entity));
  Assert.assertTrue(
      "Admin should be allowed to modify",
      timelineACLsManager.checkAccess(
          UserGroupInformation.createRemoteUser("admin"),
          ApplicationAccessType.MODIFY_APP, entity));
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity; //导入方法依赖的package包/类
public void testStorePerformance() throws IOException {
  TimelineEntity entityToStorePrep = new TimelineEntity();
  entityToStorePrep.setEntityType("TEST_ENTITY_TYPE_PREP");
  entityToStorePrep.setEntityId("TEST_ENTITY_ID_PREP");
  entityToStorePrep.setDomainId("TEST_DOMAIN");
  entityToStorePrep.addRelatedEntity("TEST_ENTITY_TYPE_2",
      "TEST_ENTITY_ID_2");
  entityToStorePrep.setStartTime(0L);

  TimelineEntities entitiesPrep = new TimelineEntities();
  entitiesPrep.addEntity(entityToStorePrep);
  store.put(entitiesPrep);

  long start = System.currentTimeMillis();
  int num = 1000000;

  Log.info("Start test for " + num);

  final String tezTaskAttemptId = "TEZ_TA";
  final String tezEntityId = "attempt_1429158534256_0001_1_00_000000_";
  final String tezTaskId = "TEZ_T";
  final String tezDomainId = "Tez_ATS_application_1429158534256_0001";

  TimelineEntity entityToStore = new TimelineEntity();
  TimelineEvent startEvt = new TimelineEvent();
  entityToStore.setEntityType(tezTaskAttemptId);

  startEvt.setEventType("TASK_ATTEMPT_STARTED");
  startEvt.setTimestamp(0);
  entityToStore.addEvent(startEvt);
  entityToStore.setDomainId(tezDomainId);

  entityToStore.addPrimaryFilter("status", "SUCCEEDED");
  entityToStore.addPrimaryFilter("applicationId",
      "application_1429158534256_0001");
  entityToStore.addPrimaryFilter("TEZ_VERTEX_ID",
      "vertex_1429158534256_0001_1_00");
  entityToStore.addPrimaryFilter("TEZ_DAG_ID", "dag_1429158534256_0001_1");
  entityToStore.addPrimaryFilter("TEZ_TASK_ID",
      "task_1429158534256_0001_1_00_000000");

  entityToStore.setStartTime(0L);
  entityToStore.addOtherInfo("startTime", 0);
  entityToStore.addOtherInfo("inProgressLogsURL",
      "localhost:8042/inProgressLogsURL");
  entityToStore.addOtherInfo("completedLogsURL", "");
  entityToStore.addOtherInfo("nodeId", "localhost:54450");
  entityToStore.addOtherInfo("nodeHttpAddress", "localhost:8042");
  entityToStore.addOtherInfo("containerId",
      "container_1429158534256_0001_01_000002");
  entityToStore.addOtherInfo("status", "RUNNING");
  entityToStore.addRelatedEntity(tezTaskId, "TEZ_TASK_ID_1");

  TimelineEntities entities = new TimelineEntities();
  entities.addEntity(entityToStore);

  for (int i = 0; i < num; ++i) {
    entityToStore.setEntityId(tezEntityId + i);
    store.put(entities);
  }

  long duration = System.currentTimeMillis() - start;
  Log.info("Duration for " + num + ": " + duration);
}