Java TimelineEntities.addEntity方法代码示例

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; //导入方法依赖的package包/类
@Test
public void testPostEntitiesWithPrimaryFilter() throws Exception {
  TimelineEntities entities = new TimelineEntities();
  TimelineEntity entity = new TimelineEntity();
  Map<String, Set<Object>> filters = new HashMap<String, Set<Object>>();
  filters.put(TimelineStore.SystemFilter.ENTITY_OWNER.toString(),
      new HashSet<Object>());
  entity.setPrimaryFilters(filters);
  entity.setEntityId("test id 6");
  entity.setEntityType("test type 6");
  entity.setStartTime(System.currentTimeMillis());
  entities.addEntity(entity);
  WebResource r = resource();
  ClientResponse response = r.path("ws").path("v1").path("timeline")
      .queryParam("user.name", "tester")
      .accept(MediaType.APPLICATION_JSON)
      .type(MediaType.APPLICATION_JSON)
      .post(ClientResponse.class, entities);
  TimelinePutResponse putResposne =
      response.getEntity(TimelinePutResponse.class);
  Assert.assertEquals(0, putResposne.getErrors().size());
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; //导入方法依赖的package包/类
@Test
public void testRelatingToNonExistingEntity() throws IOException {
  TimelineEntity entityToStore = new TimelineEntity();
  entityToStore.setEntityType("TEST_ENTITY_TYPE_1");
  entityToStore.setEntityId("TEST_ENTITY_ID_1");
  entityToStore.setDomainId(TimelineDataManager.DEFAULT_DOMAIN_ID);
  entityToStore.addRelatedEntity("TEST_ENTITY_TYPE_2", "TEST_ENTITY_ID_2");
  TimelineEntities entities = new TimelineEntities();
  entities.addEntity(entityToStore);
  store.put(entities);
  TimelineEntity entityToGet =
      store.getEntity("TEST_ENTITY_ID_2", "TEST_ENTITY_TYPE_2", null);
  Assert.assertNotNull(entityToGet);
  Assert.assertEquals("DEFAULT", entityToGet.getDomainId());
  Assert.assertEquals("TEST_ENTITY_TYPE_1",
      entityToGet.getRelatedEntities().keySet().iterator().next());
  Assert.assertEquals("TEST_ENTITY_ID_1",
      entityToGet.getRelatedEntities().values().iterator().next()
          .iterator().next());
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; //导入方法依赖的package包/类
@Test
public void testPostIncompleteEntities() throws Exception {
  TimelineEntities entities = new TimelineEntities();
  TimelineEntity entity1 = new TimelineEntity();
  entity1.setEntityId("test id 1");
  entity1.setEntityType("test type 1");
  entity1.setStartTime(System.currentTimeMillis());
  entity1.setDomainId("domain_id_1");
  entities.addEntity(entity1);
  // Add an entity with no id or type.
  entities.addEntity(new TimelineEntity());
  WebResource r = resource();
  // One of the entities has no id or type. HTTP 400 will be returned
  ClientResponse response = r.path("ws").path("v1").path("timeline")
      .queryParam("user.name", "tester").accept(MediaType.APPLICATION_JSON)
       .type(MediaType.APPLICATION_JSON).post(ClientResponse.class, entities);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  assertEquals(ClientResponse.Status.BAD_REQUEST,
      response.getClientResponseStatus());
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; //导入方法依赖的package包/类
@Test
public void testRelatingToEntityInSamePut() throws IOException {
  TimelineEntity entityToRelate = new TimelineEntity();
  entityToRelate.setEntityType("TEST_ENTITY_TYPE_2");
  entityToRelate.setEntityId("TEST_ENTITY_ID_2");
  entityToRelate.setDomainId("TEST_DOMAIN");
  TimelineEntity entityToStore = new TimelineEntity();
  entityToStore.setEntityType("TEST_ENTITY_TYPE_1");
  entityToStore.setEntityId("TEST_ENTITY_ID_1");
  entityToStore.setDomainId("TEST_DOMAIN");
  entityToStore.addRelatedEntity("TEST_ENTITY_TYPE_2", "TEST_ENTITY_ID_2");
  TimelineEntities entities = new TimelineEntities();
  entities.addEntity(entityToStore);
  entities.addEntity(entityToRelate);
  store.put(entities);
  TimelineEntity entityToGet =
      store.getEntity("TEST_ENTITY_ID_2", "TEST_ENTITY_TYPE_2", null);
  Assert.assertNotNull(entityToGet);
  Assert.assertEquals("TEST_DOMAIN", entityToGet.getDomainId());
  Assert.assertEquals("TEST_ENTITY_TYPE_1",
      entityToGet.getRelatedEntities().keySet().iterator().next());
  Assert.assertEquals("TEST_ENTITY_ID_1",
      entityToGet.getRelatedEntities().values().iterator().next()
          .iterator().next());
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; //导入方法依赖的package包/类
private void createTestFiles(ApplicationId appId, Path attemptDirPath)
    throws IOException {
  TimelineEntities entities = PluginStoreTestUtils.generateTestEntities();
  PluginStoreTestUtils.writeEntities(entities,
      new Path(attemptDirPath, TEST_SUMMARY_LOG_FILE_NAME), fs);
  Map<String, Set<Object>> primaryFilters = new HashMap<>();
  Set<Object> appSet = new HashSet<Object>();
  appSet.add(appId.toString());
  primaryFilters.put(EntityGroupPlugInForTest.APP_ID_FILTER_NAME, appSet);
  entityNew = PluginStoreTestUtils
      .createEntity(appId.toString(), "type_3", 789L, null, null,
          primaryFilters, null, "domain_id_1");
  TimelineEntities entityList = new TimelineEntities();
  entityList.addEntity(entityNew);
  PluginStoreTestUtils.writeEntities(entityList,
      new Path(attemptDirPath, mainEntityLogFileName), fs);

  FSDataOutputStream out = fs.create(
      new Path(attemptDirPath, TEST_DOMAIN_LOG_FILE_NAME));
  out.close();
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; //导入方法依赖的package包/类
@BeforeClass
public static void prepareStore() throws Exception {
  store = createStore(SCALE);
  TimelineEntities entities = new TimelineEntities();
  entities.addEntity(createApplicationTimelineEntity(
      ApplicationId.newInstance(0, SCALE + 1), true, true, false));
  entities.addEntity(createApplicationTimelineEntity(
      ApplicationId.newInstance(0, SCALE + 2), true, false, true));
  store.put(entities);
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; //导入方法依赖的package包/类
@Test
public void testPostEntities() throws Exception {
  TimelineEntities entities = new TimelineEntities();
  TimelineEntity entity = new TimelineEntity();
  entity.setEntityId("test id 1");
  entity.setEntityType("test type 1");
  entity.setStartTime(System.currentTimeMillis());
  entity.setDomainId("domain_id_1");
  entities.addEntity(entity);
  WebResource r = resource();
  // No owner, will be rejected
  ClientResponse response = r.path("ws").path("v1").path("timeline")
      .accept(MediaType.APPLICATION_JSON)
      .type(MediaType.APPLICATION_JSON)
      .post(ClientResponse.class, entities);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  assertEquals(ClientResponse.Status.FORBIDDEN,
      response.getClientResponseStatus());

  response = r.path("ws").path("v1").path("timeline")
      .queryParam("user.name", "tester")
      .accept(MediaType.APPLICATION_JSON)
      .type(MediaType.APPLICATION_JSON)
      .post(ClientResponse.class, entities);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  TimelinePutResponse putResposne =
      response.getEntity(TimelinePutResponse.class);
  Assert.assertNotNull(putResposne);
  Assert.assertEquals(0, putResposne.getErrors().size());
  // verify the entity exists in the store
  response = r.path("ws").path("v1").path("timeline")
      .path("test type 1").path("test id 1")
      .accept(MediaType.APPLICATION_JSON)
      .get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  entity = response.getEntity(TimelineEntity.class);
  Assert.assertNotNull(entity);
  Assert.assertEquals("test id 1", entity.getEntityId());
  Assert.assertEquals("test type 1", entity.getEntityType());
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; //导入方法依赖的package包/类
@Test
public void testPostEntitiesToDefaultDomain() throws Exception {
  AdminACLsManager oldAdminACLsManager =
      timelineACLsManager.setAdminACLsManager(adminACLsManager);
  try {
    TimelineEntities entities = new TimelineEntities();
    TimelineEntity entity = new TimelineEntity();
    entity.setEntityId("test id 7");
    entity.setEntityType("test type 7");
    entity.setStartTime(System.currentTimeMillis());
    entities.addEntity(entity);
    WebResource r = resource();
    ClientResponse response = r.path("ws").path("v1").path("timeline")
        .queryParam("user.name", "anybody_1")
        .accept(MediaType.APPLICATION_JSON)
        .type(MediaType.APPLICATION_JSON)
        .post(ClientResponse.class, entities);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    TimelinePutResponse putResposne =
        response.getEntity(TimelinePutResponse.class);
    Assert.assertNotNull(putResposne);
    Assert.assertEquals(0, putResposne.getErrors().size());
    // verify the entity exists in the store
    response = r.path("ws").path("v1").path("timeline")
        .path("test type 7").path("test id 7")
        .queryParam("user.name", "any_body_2")
        .accept(MediaType.APPLICATION_JSON)
        .get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    entity = response.getEntity(TimelineEntity.class);
    Assert.assertNotNull(entity);
    Assert.assertEquals("test id 7", entity.getEntityId());
    Assert.assertEquals("test type 7", entity.getEntityType());
    Assert.assertEquals(TimelineDataManager.DEFAULT_DOMAIN_ID,
        entity.getDomainId());
  } finally {
    timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
  }
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; //导入方法依赖的package包/类
public TimelinePutResponse putEntities(
    TimelineEntity... entities) throws IOException, YarnException {
  TimelineEntities entitiesContainer = new TimelineEntities();
  for (TimelineEntity entity : entities) {
    if (entity.getEntityId() == null || entity.getEntityType() == null) {
      throw new YarnException("Incomplete entity without entity id/type");
    }
    entitiesContainer.addEntity(entity);
  }
  ClientResponse resp = doPosting(entitiesContainer, null);
  return resp.getEntity(TimelinePutResponse.class);
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; //导入方法依赖的package包/类
@BeforeClass
public static void prepareStore() throws Exception {
  store = createStore(SCALE);
  TimelineEntities entities = new TimelineEntities();
  entities.addEntity(createApplicationTimelineEntity(
      ApplicationId.newInstance(0, SCALE + 1), true, true, false, false));
  entities.addEntity(createApplicationTimelineEntity(
      ApplicationId.newInstance(0, SCALE + 2), true, false, true, false));
  store.put(entities);
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; //导入方法依赖的package包/类
@Test
public void testParseEntity() throws Exception {
  // Load test data
  TimelineDataManager tdm = PluginStoreTestUtils.getTdmWithMemStore(config);
  EntityLogInfo testLogInfo = new EntityLogInfo(TEST_ATTEMPT_DIR_NAME,
      TEST_ENTITY_FILE_NAME,
      UserGroupInformation.getLoginUser().getUserName());
  testLogInfo.parseForStore(tdm, getTestRootPath(), true, jsonFactory, objMapper,
      fs);
  // Verify for the first batch
  PluginStoreTestUtils.verifyTestEntities(tdm);
  // Load new data
  TimelineEntity entityNew = PluginStoreTestUtils
      .createEntity("id_3", "type_3", 789l, null, null,
          null, null, "domain_id_1");
  TimelineEntities entityList = new TimelineEntities();
  entityList.addEntity(entityNew);
  writeEntitiesLeaveOpen(entityList,
      new Path(getTestRootPath(TEST_ATTEMPT_DIR_NAME), TEST_ENTITY_FILE_NAME));
  testLogInfo.parseForStore(tdm, getTestRootPath(), true, jsonFactory, objMapper,
      fs);
  // Verify the newly added data
  TimelineEntity entity3 = tdm.getEntity(entityNew.getEntityType(),
      entityNew.getEntityId(), EnumSet.allOf(TimelineReader.Field.class),
      UserGroupInformation.getLoginUser());
  assertNotNull(entity3);
  assertEquals("Failed to read out entity new",
      entityNew.getStartTime(), entity3.getStartTime());
  tdm.close();
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; //导入方法依赖的package包/类
@Test
public void testPostEntities() throws Exception {
  TimelineEntities entities = new TimelineEntities();
  TimelineEntity entity = new TimelineEntity();
  entity.setEntityId("test id");
  entity.setEntityType("test type");
  entity.setStartTime(System.currentTimeMillis());
  entities.addEntity(entity);
  WebResource r = resource();
  ClientResponse response = r.path("ws").path("v1").path("timeline")
      .accept(MediaType.APPLICATION_JSON)
      .type(MediaType.APPLICATION_JSON)
      .post(ClientResponse.class, entities);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  TimelinePutResponse putResposne = response.getEntity(TimelinePutResponse.class);
  Assert.assertNotNull(putResposne);
  Assert.assertEquals(0, putResposne.getErrors().size());
  // verify the entity exists in the store
  response = r.path("ws").path("v1").path("timeline")
      .path("test type").path("test id")
      .accept(MediaType.APPLICATION_JSON)
      .get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  entity = response.getEntity(TimelineEntity.class);
  Assert.assertNotNull(entity);
  Assert.assertEquals("test id", entity.getEntityId());
  Assert.assertEquals("test type", entity.getEntityType());
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; //导入方法依赖的package包/类
@Test
public void testGetEntityWithYarnACLsEnabled() throws Exception {
  AdminACLsManager oldAdminACLsManager =
      timelineACLsManager.setAdminACLsManager(adminACLsManager);
  try {
    TimelineEntities entities = new TimelineEntities();
    TimelineEntity entity = new TimelineEntity();
    entity.setEntityId("test id 3");
    entity.setEntityType("test type 3");
    entity.setStartTime(System.currentTimeMillis());
    entity.setDomainId("domain_id_1");
    entities.addEntity(entity);
    WebResource r = resource();
    ClientResponse response = r.path("ws").path("v1").path("timeline")
        .queryParam("user.name", "writer_user_1")
        .accept(MediaType.APPLICATION_JSON)
        .type(MediaType.APPLICATION_JSON)
        .post(ClientResponse.class, entities);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    TimelinePutResponse putResponse =
        response.getEntity(TimelinePutResponse.class);
    Assert.assertEquals(0, putResponse.getErrors().size());
    // verify the system data will not be exposed
    // 1. No field specification
    response = r.path("ws").path("v1").path("timeline")
        .path("test type 3").path("test id 3")
        .queryParam("user.name", "reader_user_1")
        .accept(MediaType.APPLICATION_JSON)
        .get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    entity = response.getEntity(TimelineEntity.class);
    Assert.assertNull(entity.getPrimaryFilters().get(
        TimelineStore.SystemFilter.ENTITY_OWNER.toString()));
    // 2. other field
    response = r.path("ws").path("v1").path("timeline")
        .path("test type 3").path("test id 3")
        .queryParam("fields", "relatedentities")
        .queryParam("user.name", "reader_user_1")
        .accept(MediaType.APPLICATION_JSON)
        .get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    entity = response.getEntity(TimelineEntity.class);
    Assert.assertNull(entity.getPrimaryFilters().get(
        TimelineStore.SystemFilter.ENTITY_OWNER.toString()));
    // 3. primaryfilters field
    response = r.path("ws").path("v1").path("timeline")
        .path("test type 3").path("test id 3")
        .queryParam("fields", "primaryfilters")
        .queryParam("user.name", "reader_user_1")
        .accept(MediaType.APPLICATION_JSON)
        .get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    entity = response.getEntity(TimelineEntity.class);
    Assert.assertNull(entity.getPrimaryFilters().get(
        TimelineStore.SystemFilter.ENTITY_OWNER.toString()));

    // get entity with other user
    response = r.path("ws").path("v1").path("timeline")
        .path("test type 3").path("test id 3")
        .queryParam("user.name", "reader_user_2")
        .accept(MediaType.APPLICATION_JSON)
        .get(ClientResponse.class);
    assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
    assertEquals(ClientResponse.Status.NOT_FOUND,
        response.getClientResponseStatus());
  } finally {
    timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
  }
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; //导入方法依赖的package包/类
private TimelinePutResponse doPostEntities(
    TimelineEntities entities,
    UserGroupInformation callerUGI) throws YarnException, IOException {
  if (entities == null) {
    return new TimelinePutResponse();
  }
  metrics.incrPostEntitiesTotal(entities.getEntities().size());
  TimelineEntities entitiesToPut = new TimelineEntities();
  List<TimelinePutResponse.TimelinePutError> errors =
      new ArrayList<TimelinePutResponse.TimelinePutError>();
  for (TimelineEntity entity : entities.getEntities()) {

    // if the domain id is not specified, the entity will be put into
    // the default domain
    if (entity.getDomainId() == null ||
        entity.getDomainId().length() == 0) {
      entity.setDomainId(DEFAULT_DOMAIN_ID);
    }
    if (entity.getEntityId() == null || entity.getEntityType() == null) {
      throw new BadRequestException("Incomplete entity without entity"
          + " id/type");
    }
    // check if there is existing entity
    TimelineEntity existingEntity = null;
    try {
      existingEntity =
          store.getEntity(entity.getEntityId(), entity.getEntityType(),
              EnumSet.of(Field.PRIMARY_FILTERS));
      if (existingEntity != null) {
        addDefaultDomainIdIfAbsent(existingEntity);
        if (!existingEntity.getDomainId().equals(entity.getDomainId())) {
          throw new YarnException("The domain of the timeline entity "
            + "{ id: " + entity.getEntityId() + ", type: "
            + entity.getEntityType() + " } is not allowed to be changed from "
            + existingEntity.getDomainId() + " to " + entity.getDomainId());
        }
      }
      if (!timelineACLsManager.checkAccess(
          callerUGI, ApplicationAccessType.MODIFY_APP, entity)) {
        throw new YarnException(callerUGI
            + " is not allowed to put the timeline entity "
            + "{ id: " + entity.getEntityId() + ", type: "
            + entity.getEntityType() + " } into the domain "
            + entity.getDomainId() + ".");
      }
    } catch (Exception e) {
      // Skip the entity which already exists and was put by others
      LOG.warn("Skip the timeline entity: { id: " + entity.getEntityId()
          + ", type: "+ entity.getEntityType() + " }", e);
      TimelinePutResponse.TimelinePutError error =
          new TimelinePutResponse.TimelinePutError();
      error.setEntityId(entity.getEntityId());
      error.setEntityType(entity.getEntityType());
      error.setErrorCode(
          TimelinePutResponse.TimelinePutError.ACCESS_DENIED);
      errors.add(error);
      continue;
    }

    entitiesToPut.addEntity(entity);
  }

  TimelinePutResponse response = store.put(entitiesToPut);
  // add the errors of timeline system filter key conflict
  response.addErrors(errors);
  return response;
}
 

import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities; //导入方法依赖的package包/类
public void testStorePerformance() throws IOException {
  TimelineEntity entityToStorePrep = new TimelineEntity();
  entityToStorePrep.setEntityType("TEST_ENTITY_TYPE_PREP");
  entityToStorePrep.setEntityId("TEST_ENTITY_ID_PREP");
  entityToStorePrep.setDomainId("TEST_DOMAIN");
  entityToStorePrep.addRelatedEntity("TEST_ENTITY_TYPE_2",
      "TEST_ENTITY_ID_2");
  entityToStorePrep.setStartTime(0L);

  TimelineEntities entitiesPrep = new TimelineEntities();
  entitiesPrep.addEntity(entityToStorePrep);
  store.put(entitiesPrep);

  long start = System.currentTimeMillis();
  int num = 1000000;

  Log.info("Start test for " + num);

  final String tezTaskAttemptId = "TEZ_TA";
  final String tezEntityId = "attempt_1429158534256_0001_1_00_000000_";
  final String tezTaskId = "TEZ_T";
  final String tezDomainId = "Tez_ATS_application_1429158534256_0001";

  TimelineEntity entityToStore = new TimelineEntity();
  TimelineEvent startEvt = new TimelineEvent();
  entityToStore.setEntityType(tezTaskAttemptId);

  startEvt.setEventType("TASK_ATTEMPT_STARTED");
  startEvt.setTimestamp(0);
  entityToStore.addEvent(startEvt);
  entityToStore.setDomainId(tezDomainId);

  entityToStore.addPrimaryFilter("status", "SUCCEEDED");
  entityToStore.addPrimaryFilter("applicationId",
      "application_1429158534256_0001");
  entityToStore.addPrimaryFilter("TEZ_VERTEX_ID",
      "vertex_1429158534256_0001_1_00");
  entityToStore.addPrimaryFilter("TEZ_DAG_ID", "dag_1429158534256_0001_1");
  entityToStore.addPrimaryFilter("TEZ_TASK_ID",
      "task_1429158534256_0001_1_00_000000");

  entityToStore.setStartTime(0L);
  entityToStore.addOtherInfo("startTime", 0);
  entityToStore.addOtherInfo("inProgressLogsURL",
      "localhost:8042/inProgressLogsURL");
  entityToStore.addOtherInfo("completedLogsURL", "");
  entityToStore.addOtherInfo("nodeId", "localhost:54450");
  entityToStore.addOtherInfo("nodeHttpAddress", "localhost:8042");
  entityToStore.addOtherInfo("containerId",
      "container_1429158534256_0001_01_000002");
  entityToStore.addOtherInfo("status", "RUNNING");
  entityToStore.addRelatedEntity(tezTaskId, "TEZ_TASK_ID_1");

  TimelineEntities entities = new TimelineEntities();
  entities.addEntity(entityToStore);

  for (int i = 0; i < num; ++i) {
    entityToStore.setEntityId(tezEntityId + i);
    store.put(entities);
  }

  long duration = System.currentTimeMillis() - start;
  Log.info("Duration for " + num + ": " + duration);
}