Java ContainerModel类代码示例

import org.apache.samza.job.model.ContainerModel; //导入依赖的package包/类
@Bean
@ConditionalOnMissingBean
public SamzaContainer samzaContainer() {

    JobModel jobModel = samzaJobModel();

    int containerId = samzaContainerId();
    Assert.isTrue(containerId >= 0, "samzaContainerId must be a non-negative integer (0 or greater).");

    Map<Integer, ContainerModel> containers = jobModel.getContainers();
    ContainerModel containerModel = containers.get(containerId);
    if (containerModel == null) {
        String msg = "Container model does not exist for samza container id '" + containerId + "'.  " +
            "Ensure that samzaContainerId is a zero-based integer less than the total number " +
            "of containers for the job.  Number of containers in the model: " + containers.size();
        throw new BeanInitializationException(msg);
    }

    return SamzaContainer$.MODULE$.apply(containerModel, jobModel);
}
 

import org.apache.samza.job.model.ContainerModel; //导入依赖的package包/类
/**
 * Generate new JobModel when becoming a leader or the list of processor changed.
 */
private JobModel generateNewJobModel(List<String> processors) {
  String zkJobModelVersion = zkUtils.getJobModelVersion();
  // If JobModel exists in zookeeper && cached JobModel version is unequal to JobModel version stored in zookeeper.
  if (zkJobModelVersion != null && !Objects.equals(cachedJobModelVersion, zkJobModelVersion)) {
    JobModel jobModel = zkUtils.getJobModel(zkJobModelVersion);
    for (ContainerModel containerModel : jobModel.getContainers().values()) {
      containerModel.getTasks().forEach((taskName, taskModel) -> changeLogPartitionMap.put(taskName, taskModel.getChangelogPartition().getPartitionId()));
    }
    cachedJobModelVersion = zkJobModelVersion;
  }
  /**
   * Host affinity is not supported in standalone. Hence, LocalityManager(which is responsible for container
   * to host mapping) is passed in as null when building the jobModel.
   */
  return JobModelManager.readJobModel(this.config, changeLogPartitionMap, null, streamMetadataCache, processors);
}
 

import org.apache.samza.job.model.ContainerModel; //导入依赖的package包/类
/**
 * Note: This is artificial. It makes assumptions about task grouping, changelog partition mapping, etc.
 * that will likely not be true at runtime, but are not the focus of this test.
 */
public static Set<ContainerModel> generateContainerModels(int numContainers, int taskCount) {
  Set<ContainerModel> models = new HashSet<>(numContainers);
  int[] taskCountPerContainer = calculateTaskCountPerContainer(taskCount, numContainers);
  int j = 0;
  for (int i = 0; i < numContainers; i++) {
    int[] partitions = new int[taskCountPerContainer[i]];
    for (int k = 0; k < taskCountPerContainer[i]; k++) {
      partitions[k] = j + k;
    }
    j += taskCountPerContainer[i];

    models.add(createContainerModel(String.valueOf(i), partitions));
  }
  return models;
}
 

import org.apache.samza.job.model.ContainerModel; //导入依赖的package包/类
@Test
public void testGroupHappyPath() {
  Set<TaskModel> taskModels = generateTaskModels(5);

  Set<ContainerModel> containers = new GroupByContainerCount(2).group(taskModels);

  Map<String, ContainerModel> containersMap = new HashMap<>();
  for (ContainerModel container : containers) {
    containersMap.put(container.getProcessorId(), container);
  }

  assertEquals(2, containers.size());
  ContainerModel container0 = containersMap.get("0");
  ContainerModel container1 = containersMap.get("1");
  assertNotNull(container0);
  assertNotNull(container1);
  assertEquals("0", container0.getProcessorId());
  assertEquals("1", container1.getProcessorId());
  assertEquals(3, container0.getTasks().size());
  assertEquals(2, container1.getTasks().size());
  assertTrue(container0.getTasks().containsKey(getTaskName(0)));
  assertTrue(container0.getTasks().containsKey(getTaskName(2)));
  assertTrue(container0.getTasks().containsKey(getTaskName(4)));
  assertTrue(container1.getTasks().containsKey(getTaskName(1)));
  assertTrue(container1.getTasks().containsKey(getTaskName(3)));
}
 

import org.apache.samza.job.model.ContainerModel; //导入依赖的package包/类
@Test
public void testBalancerOldContainerCountOne() {
  Set<TaskModel> taskModels = generateTaskModels(3);
  Set<ContainerModel> prevContainers = new GroupByContainerCount(1).group(taskModels);
  Map<String, String> prevTaskToContainerMapping = generateTaskContainerMapping(prevContainers);
  when(taskAssignmentManager.readTaskAssignment()).thenReturn(prevTaskToContainerMapping);

  Set<ContainerModel> groupContainers = new GroupByContainerCount(3).group(taskModels);
  Set<ContainerModel> balanceContainers = new GroupByContainerCount(3).balance(taskModels, localityManager);

  // Results should be the same as calling group()
  assertEquals(groupContainers, balanceContainers);

  // Verify task mappings are saved
  verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(0).getTaskName(), "0");
  verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(1).getTaskName(), "1");
  verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(2).getTaskName(), "2");

  verify(taskAssignmentManager, never()).deleteTaskContainerMappings(anyCollection());
}
 

import org.apache.samza.job.model.ContainerModel; //导入依赖的package包/类
@Test
public void testBalancerNewContainerCountOne() {
  Set<TaskModel> taskModels = generateTaskModels(3);
  Set<ContainerModel> prevContainers = new GroupByContainerCount(3).group(taskModels);
  Map<String, String> prevTaskToContainerMapping = generateTaskContainerMapping(prevContainers);
  when(taskAssignmentManager.readTaskAssignment()).thenReturn(prevTaskToContainerMapping);

  Set<ContainerModel> groupContainers = new GroupByContainerCount(1).group(taskModels);
  Set<ContainerModel> balanceContainers = new GroupByContainerCount(1).balance(taskModels, localityManager);

  // Results should be the same as calling group()
  assertEquals(groupContainers, balanceContainers);

  verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(0).getTaskName(), "0");
  verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(1).getTaskName(), "0");
  verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(2).getTaskName(), "0");

  verify(taskAssignmentManager, never()).deleteTaskContainerMappings(anyCollection());
}
 

import org.apache.samza.job.model.ContainerModel; //导入依赖的package包/类
@Test
public void testBalancerEmptyTaskMapping() {
  Set<TaskModel> taskModels = generateTaskModels(3);
  when(taskAssignmentManager.readTaskAssignment()).thenReturn(new HashMap<String, String>());

  Set<ContainerModel> groupContainers = new GroupByContainerCount(1).group(taskModels);
  Set<ContainerModel> balanceContainers = new GroupByContainerCount(1).balance(taskModels, localityManager);

  // Results should be the same as calling group()
  assertEquals(groupContainers, balanceContainers);

  verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(0).getTaskName(), "0");
  verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(1).getTaskName(), "0");
  verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(2).getTaskName(), "0");

  verify(taskAssignmentManager, never()).deleteTaskContainerMappings(anyCollection());
}
 

import org.apache.samza.job.model.ContainerModel; //导入依赖的package包/类
@Test
public void testGroupTaskCountIncrease() {
  int taskCount = 3;
  Set<TaskModel> taskModels = generateTaskModels(taskCount);
  Set<ContainerModel> prevContainers = new GroupByContainerCount(2).group(generateTaskModels(taskCount - 1)); // Here's the key step
  Map<String, String> prevTaskToContainerMapping = generateTaskContainerMapping(prevContainers);
  when(taskAssignmentManager.readTaskAssignment()).thenReturn(prevTaskToContainerMapping);

  Set<ContainerModel> groupContainers = new GroupByContainerCount(1).group(taskModels);
  Set<ContainerModel> balanceContainers = new GroupByContainerCount(1).balance(taskModels, localityManager);

  // Results should be the same as calling group()
  assertEquals(groupContainers, balanceContainers);

  verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(0).getTaskName(), "0");
  verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(1).getTaskName(), "0");
  verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(2).getTaskName(), "0");

  verify(taskAssignmentManager).deleteTaskContainerMappings(anyCollection());
}
 

import org.apache.samza.job.model.ContainerModel; //导入依赖的package包/类
@Test
public void testGroupTaskCountDecrease() {
  int taskCount = 3;
  Set<TaskModel> taskModels = generateTaskModels(taskCount);
  Set<ContainerModel> prevContainers = new GroupByContainerCount(3).group(generateTaskModels(taskCount + 1)); // Here's the key step
  Map<String, String> prevTaskToContainerMapping = generateTaskContainerMapping(prevContainers);
  when(taskAssignmentManager.readTaskAssignment()).thenReturn(prevTaskToContainerMapping);

  Set<ContainerModel> groupContainers = new GroupByContainerCount(1).group(taskModels);
  Set<ContainerModel> balanceContainers = new GroupByContainerCount(1).balance(taskModels, localityManager);

  // Results should be the same as calling group()
  assertEquals(groupContainers, balanceContainers);

  verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(0).getTaskName(), "0");
  verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(1).getTaskName(), "0");
  verify(taskAssignmentManager).writeTaskContainerMapping(getTaskName(2).getTaskName(), "0");

  verify(taskAssignmentManager).deleteTaskContainerMappings(anyCollection());
}
 

import org.apache.samza.job.model.ContainerModel; //导入依赖的package包/类
@Test
public void testGroupHappyPath() {
  Set<TaskModel> taskModels = generateTaskModels(5);

  Set<ContainerModel> containers = buildSimpleGrouper(2).group(taskModels);

  Map<String, ContainerModel> containersMap = new HashMap<>();
  for (ContainerModel container : containers) {
    containersMap.put(container.getProcessorId(), container);
  }

  assertEquals(2, containers.size());
  ContainerModel container0 = containersMap.get("0");
  ContainerModel container1 = containersMap.get("1");
  assertNotNull(container0);
  assertNotNull(container1);
  assertEquals("0", container0.getProcessorId());
  assertEquals("1", container1.getProcessorId());
  assertEquals(3, container0.getTasks().size());
  assertEquals(2, container1.getTasks().size());
  assertTrue(container0.getTasks().containsKey(getTaskName(0)));
  assertTrue(container0.getTasks().containsKey(getTaskName(2)));
  assertTrue(container0.getTasks().containsKey(getTaskName(4)));
  assertTrue(container1.getTasks().containsKey(getTaskName(1)));
  assertTrue(container1.getTasks().containsKey(getTaskName(3)));
}
 

import org.apache.samza.job.model.ContainerModel; //导入依赖的package包/类
@Test
public void testGroupWithNullContainerIds() {
  Set<TaskModel> taskModels = generateTaskModels(5);

  Set<ContainerModel> containers = buildSimpleGrouper(2).group(taskModels, null);

  Map<String, ContainerModel> containersMap = new HashMap<>();
  for (ContainerModel container : containers) {
    containersMap.put(container.getProcessorId(), container);
  }

  assertEquals(2, containers.size());
  ContainerModel container0 = containersMap.get("0");
  ContainerModel container1 = containersMap.get("1");
  assertNotNull(container0);
  assertNotNull(container1);
  assertEquals("0", container0.getProcessorId());
  assertEquals("1", container1.getProcessorId());
  assertEquals(3, container0.getTasks().size());
  assertEquals(2, container1.getTasks().size());
  assertTrue(container0.getTasks().containsKey(getTaskName(0)));
  assertTrue(container0.getTasks().containsKey(getTaskName(2)));
  assertTrue(container0.getTasks().containsKey(getTaskName(4)));
  assertTrue(container1.getTasks().containsKey(getTaskName(1)));
  assertTrue(container1.getTasks().containsKey(getTaskName(3)));
}
 

import org.apache.samza.job.model.ContainerModel; //导入依赖的package包/类
@Test
public void testFewerTasksThanContainers() {
  final String testContainerId1 = "1";
  final String testContainerId2 = "2";
  final int testProcessorId = 1;

  Set<TaskModel> taskModels = generateTaskModels(1);
  List<String> containerIds = ImmutableList.of(testContainerId1, testContainerId2);

  Map<TaskName, TaskModel> expectedTasks = taskModels.stream()
                                                     .collect(Collectors.toMap(TaskModel::getTaskName, x -> x));
  ContainerModel expectedContainerModel = new ContainerModel(testContainerId1, testProcessorId, expectedTasks);

  Set<ContainerModel> actualContainerModels = buildSimpleGrouper().group(taskModels, containerIds);

  assertEquals(1, actualContainerModels.size());
  assertEquals(ImmutableSet.of(expectedContainerModel), actualContainerModels);
}
 

import org.apache.samza.job.model.ContainerModel; //导入依赖的package包/类
@Before
public void setup() throws IOException {
  Map<String, String> configMap = new HashMap<String, String>();
  Set<SystemStreamPartition> ssp = new HashSet<>();
  configMap.put("a", "b");
  Config config = new MapConfig(configMap);
  TaskName taskName = new TaskName("test");
  ssp.add(new SystemStreamPartition("foo", "bar", new Partition(1)));
  TaskModel taskModel = new TaskModel(taskName, ssp, new Partition(2));
  Map<TaskName, TaskModel> tasks = new HashMap<TaskName, TaskModel>();
  tasks.put(taskName, taskModel);
  ContainerModel containerModel = new ContainerModel("1", 1, tasks);
  Map<String, ContainerModel> containerMap = new HashMap<String, ContainerModel>();
  containerMap.put("1", containerModel);
  jobModel = new JobModel(config, containerMap);
}
 

import org.apache.samza.job.model.ContainerModel; //导入依赖的package包/类
/**
 * This method returns the number of tasks in the job. It works by querying the server, and getting the job model.
 * Then it extracts the number of tasks from the job model
 *
 * @return current number of tasks in the job
 */
public int getCurrentNumTasks() {
  int currentNumTasks = 0;
  for (ContainerModel containerModel : SamzaContainer.readJobModel(coordinatorServerURL, defaultReadJobModelDelayMs).getContainers().values()) {
    currentNumTasks += containerModel.getTasks().size();
  }
  return currentNumTasks;
}
 

import org.apache.samza.job.model.ContainerModel; //导入依赖的package包/类
/**
 * Initialize this {@link OperatorImpl} and its user-defined functions.
 *
 * @param config  the {@link Config} for the task
 * @param context  the {@link TaskContext} for the task
 */
public final void init(Config config, TaskContext context) {
  String opId = getOpImplId();

  if (initialized) {
    throw new IllegalStateException(String.format("Attempted to initialize Operator %s more than once.", opId));
  }

  if (closed) {
    throw new IllegalStateException(String.format("Attempted to initialize Operator %s after it was closed.", opId));
  }

  this.highResClock = createHighResClock(config);
  registeredOperators = new HashSet<>();
  prevOperators = new HashSet<>();
  inputStreams = new HashSet<>();
  MetricsRegistry metricsRegistry = context.getMetricsRegistry();
  this.numMessage = metricsRegistry.newCounter(METRICS_GROUP, opId + "-messages");
  this.handleMessageNs = metricsRegistry.newTimer(METRICS_GROUP, opId + "-handle-message-ns");
  this.handleTimerNs = metricsRegistry.newTimer(METRICS_GROUP, opId + "-handle-timer-ns");
  this.taskName = context.getTaskName();

  TaskContextImpl taskContext = (TaskContextImpl) context;
  this.eosStates = (EndOfStreamStates) taskContext.fetchObject(EndOfStreamStates.class.getName());
  this.watermarkStates = (WatermarkStates) taskContext.fetchObject(WatermarkStates.class.getName());

  if (taskContext.getJobModel() != null) {
    ContainerModel containerModel = taskContext.getJobModel().getContainers()
        .get(context.getSamzaContainerContext().id);
    this.taskModel = containerModel.getTasks().get(taskName);
  } else {
    this.taskModel = null;
    this.usedInCurrentTask = true;
  }

  handleInit(config, context);

  initialized = true;
}