Java JobModel类代码示例

import org.apache.samza.job.model.JobModel; //导入依赖的package包/类
@Bean
@ConditionalOnMissingBean
public SamzaContainer samzaContainer() {

    JobModel jobModel = samzaJobModel();

    int containerId = samzaContainerId();
    Assert.isTrue(containerId >= 0, "samzaContainerId must be a non-negative integer (0 or greater).");

    Map<Integer, ContainerModel> containers = jobModel.getContainers();
    ContainerModel containerModel = containers.get(containerId);
    if (containerModel == null) {
        String msg = "Container model does not exist for samza container id '" + containerId + "'.  " +
            "Ensure that samzaContainerId is a zero-based integer less than the total number " +
            "of containers for the job.  Number of containers in the model: " + containers.size();
        throw new BeanInitializationException(msg);
    }

    return SamzaContainer$.MODULE$.apply(containerModel, jobModel);
}
 

import org.apache.samza.job.model.JobModel; //导入依赖的package包/类
/**
 * Writes the job model to the blob.
 * Write is successful only if the lease ID passed is valid and the processor holds the lease.
 * Called by the leader.
 * @param prevJM Previous job model version that the processor was operating on.
 * @param currJM Current job model version that the processor is operating on.
 * @param prevJMV Previous job model version that the processor was operating on.
 * @param currJMV Current job model version that the processor is operating on.
 * @param leaseId LeaseID of the lease that the processor holds on the blob. Null if there is no lease.
 * @return true if write to the blob is successful, false if leaseID is null or an Azure storage service error or IO exception occurred.
 */
public boolean publishJobModel(JobModel prevJM, JobModel currJM, String prevJMV, String currJMV, String leaseId) {
  try {
    if (leaseId == null) {
      return false;
    }
    JobModelBundle bundle = new JobModelBundle(prevJM, currJM, prevJMV, currJMV);
    byte[] data = SamzaObjectMapper.getObjectMapper().writerWithDefaultPrettyPrinter().writeValueAsBytes(bundle);
    byte[] pageData = Arrays.copyOf(data, (int) JOB_MODEL_BLOCK_SIZE);
    InputStream is = new ByteArrayInputStream(pageData);
    blob.uploadPages(is, 0, JOB_MODEL_BLOCK_SIZE, AccessCondition.generateLeaseCondition(leaseId), null, null);
    LOG.info("Uploaded {} jobModel to blob", bundle.getCurrJobModel());
    return true;
  } catch (StorageException | IOException e) {
    LOG.error("JobModel publish failed for version = " + currJMV, e);
    return false;
  }
}
 

import org.apache.samza.job.model.JobModel; //导入依赖的package包/类
/**
 * Generate new JobModel when becoming a leader or the list of processor changed.
 */
private JobModel generateNewJobModel(List<String> processors) {
  String zkJobModelVersion = zkUtils.getJobModelVersion();
  // If JobModel exists in zookeeper && cached JobModel version is unequal to JobModel version stored in zookeeper.
  if (zkJobModelVersion != null && !Objects.equals(cachedJobModelVersion, zkJobModelVersion)) {
    JobModel jobModel = zkUtils.getJobModel(zkJobModelVersion);
    for (ContainerModel containerModel : jobModel.getContainers().values()) {
      containerModel.getTasks().forEach((taskName, taskModel) -> changeLogPartitionMap.put(taskName, taskModel.getChangelogPartition().getPartitionId()));
    }
    cachedJobModelVersion = zkJobModelVersion;
  }
  /**
   * Host affinity is not supported in standalone. Hence, LocalityManager(which is responsible for container
   * to host mapping) is passed in as null when building the jobModel.
   */
  return JobModelManager.readJobModel(this.config, changeLogPartitionMap, null, streamMetadataCache, processors);
}
 

import org.apache.samza.job.model.JobModel; //导入依赖的package包/类
@Override
public void start() {
  // No-op
  JobModel jobModel = null;
  try {
    jobModel = getJobModel();
  } catch (Exception e) {
    LOGGER.error("Exception while trying to getJobModel.", e);
    if (coordinatorListener != null) {
      coordinatorListener.onCoordinatorFailure(e);
    }
  }
  if (jobModel != null && jobModel.getContainers().containsKey(processorId)) {
    if (coordinatorListener != null) {
      coordinatorListener.onNewJobModel(processorId, jobModel);
    }
  } else {
    stop();
  }
}
 

import org.apache.samza.job.model.JobModel; //导入依赖的package包/类
public TaskContextImpl(TaskName taskName,
                       TaskInstanceMetrics metrics,
                       SamzaContainerContext containerContext,
                       Set<SystemStreamPartition> systemStreamPartitions,
                       OffsetManager offsetManager,
                       TaskStorageManager storageManager,
                       TableManager tableManager,
                       JobModel jobModel,
                       StreamMetadataCache streamMetadataCache) {
  this.taskName = taskName;
  this.metrics = metrics;
  this.containerContext = containerContext;
  this.systemStreamPartitions = ImmutableSet.copyOf(systemStreamPartitions);
  this.offsetManager = offsetManager;
  this.storageManager = storageManager;
  this.tableManager = tableManager;
  this.jobModel = jobModel;
  this.streamMetadataCache = streamMetadataCache;
}
 

import org.apache.samza.job.model.JobModel; //导入依赖的package包/类
@Test
public void testFollowerShouldStopWhenNotPartOfGeneratedJobModel() {
  ZkKeyBuilder keyBuilder = Mockito.mock(ZkKeyBuilder.class);
  ZkClient mockZkClient = Mockito.mock(ZkClient.class);
  Mockito.when(keyBuilder.getJobModelVersionBarrierPrefix()).thenReturn(TEST_BARRIER_ROOT);

  ZkUtils zkUtils = Mockito.mock(ZkUtils.class);
  Mockito.when(zkUtils.getKeyBuilder()).thenReturn(keyBuilder);
  Mockito.when(zkUtils.getZkClient()).thenReturn(mockZkClient);
  Mockito.when(zkUtils.getJobModel(TEST_JOB_MODEL_VERSION)).thenReturn(new JobModel(new MapConfig(), new HashMap<>()));

  ZkJobCoordinator zkJobCoordinator = Mockito.spy(new ZkJobCoordinator(new MapConfig(), new NoOpMetricsRegistry(), zkUtils));
  zkJobCoordinator.onNewJobModelAvailable(TEST_JOB_MODEL_VERSION);

  Mockito.verify(zkJobCoordinator, Mockito.atMost(1)).stop();
}
 

import org.apache.samza.job.model.JobModel; //导入依赖的package包/类
@Override
SamzaContainer createSamzaContainer(String processorId, JobModel jobModel) {
  if (container == null) {
    RunLoop mockRunLoop = mock(RunLoop.class);
    doAnswer(invocation ->
      {
        try {
          runLoopStartForMain.countDown();
          containerStop.await();
        } catch (InterruptedException e) {
          System.out.println("In exception" + e);
          e.printStackTrace();
        }
        return null;
      }).when(mockRunLoop).run();

    doAnswer(invocation ->
      {
        containerStop.countDown();
        return null;
      }).when(mockRunLoop).shutdown();
    container = StreamProcessorTestUtils.getDummyContainer(mockRunLoop, mock(StreamTask.class));
  }
  return container;
}
 

import org.apache.samza.job.model.JobModel; //导入依赖的package包/类
@Before
public void setup() throws IOException {
  Map<String, String> configMap = new HashMap<String, String>();
  Set<SystemStreamPartition> ssp = new HashSet<>();
  configMap.put("a", "b");
  Config config = new MapConfig(configMap);
  TaskName taskName = new TaskName("test");
  ssp.add(new SystemStreamPartition("foo", "bar", new Partition(1)));
  TaskModel taskModel = new TaskModel(taskName, ssp, new Partition(2));
  Map<TaskName, TaskModel> tasks = new HashMap<TaskName, TaskModel>();
  tasks.put(taskName, taskModel);
  ContainerModel containerModel = new ContainerModel("1", 1, tasks);
  Map<String, ContainerModel> containerMap = new HashMap<String, ContainerModel>();
  containerMap.put("1", containerModel);
  jobModel = new JobModel(config, containerMap);
}
 

import org.apache.samza.job.model.JobModel; //导入依赖的package包/类
/**
 * Critical test to guarantee compatibility between samza 0.12 container models and 0.13+
 *
 * Samza 0.12 contains only "container-id" (integer) in the ContainerModel. "processor-id" (String) is added in 0.13.
 * When serializing, we serialize both the fields in 0.13. Deserialization correctly handles the fields in 0.13.
 */
@Test
public void testContainerModelCompatible() {
  try {
    String newJobModelString = "{\"config\":{\"a\":\"b\"},\"containers\":{\"1\":{\"processor-id\":\"1\",\"container-id\":1,\"tasks\":{\"test\":{\"task-name\":\"test\",\"system-stream-partitions\":[{\"system\":\"foo\",\"partition\":1,\"stream\":\"bar\"}],\"changelog-partition\":2}}}},\"max-change-log-stream-partitions\":3,\"all-container-locality\":{\"1\":null}}";
    ObjectMapper mapper = SamzaObjectMapper.getObjectMapper();
    JobModel jobModel = mapper.readValue(newJobModelString, JobModel.class);

    String oldJobModelString = "{\"config\":{\"a\":\"b\"},\"containers\":{\"1\":{\"container-id\":1,\"tasks\":{\"test\":{\"task-name\":\"test\",\"system-stream-partitions\":[{\"system\":\"foo\",\"partition\":1,\"stream\":\"bar\"}],\"changelog-partition\":2}}}},\"max-change-log-stream-partitions\":3,\"all-container-locality\":{\"1\":null}}";
    ObjectMapper mapper1 = SamzaObjectMapper.getObjectMapper();
    JobModel jobModel1 = mapper1.readValue(oldJobModelString, JobModel.class);

    Assert.assertEquals(jobModel, jobModel1);
  } catch (Exception e) {
    e.printStackTrace();
  }
}
 

import org.apache.samza.job.model.JobModel; //导入依赖的package包/类
/**
 * get the config for the AM or containers based on the containers' names.
 *
 * @return Config the config of this container
 */
protected Config getConfig() {
  Config config;

  try {
    if (isApplicationMaster) {
      config = JobModelManager.currentJobModelManager().jobModel().getConfig();
    } else {
      String url = System.getenv(ShellCommandConfig.ENV_COORDINATOR_URL());
      config = SamzaObjectMapper.getObjectMapper()
          .readValue(Util.read(new URL(url), 30000), JobModel.class)
          .getConfig();
    }
  } catch (IOException e) {
    throw new SamzaException("can not read the config", e);
  }

  return config;
}
 

import org.apache.samza.job.model.JobModel; //导入依赖的package包/类
@Bean
@ConditionalOnMissingBean
public JobModel samzaJobModel() {
    Config samzaConfig = samzaConfig();

    int count = samzaContainerCount();
    Assert.isTrue(count > 0, "samzaContainerCount must be a positive integer (greater than zero).");

    return JobCoordinator$.MODULE$.buildJobModel(samzaConfig, count);
}
 

import org.apache.samza.job.model.JobModel; //导入依赖的package包/类
/**
 * Reads the current job model from the blob.
 * @return The current job model published on the blob. Returns null when job model details not found on the blob.
 * @throws AzureException in getJobModelBundle() if an Azure storage service error occurred.
 * @throws SamzaException in getJobModelBundle() if data retrieved from blob could not be parsed by SamzaObjectMapper.
 */
public JobModel getJobModel() {
  LOG.info("Reading the job model from blob.");
  JobModelBundle jmBundle = getJobModelBundle();
  if (jmBundle == null) {
    LOG.error("Job Model details don't exist on the blob.");
    return null;
  }
  JobModel jm = jmBundle.getCurrJobModel();
  return jm;
}
 

import org.apache.samza.job.model.JobModel; //导入依赖的package包/类
/**
 * Publishes new job model into ZK.
 * This call should FAIL if the node already exists.
 * @param jobModelVersion  version of the jobModeL to publish
 * @param jobModel jobModel to publish
 *
 */
public void publishJobModel(String jobModelVersion, JobModel jobModel) {
  try {
    ObjectMapper mmapper = SamzaObjectMapper.getObjectMapper();
    String jobModelStr = mmapper.writerWithDefaultPrettyPrinter().writeValueAsString(jobModel);
    LOG.info("jobModelAsString=" + jobModelStr);
    zkClient.createPersistent(keyBuilder.getJobModelPath(jobModelVersion), jobModelStr);
    LOG.info("wrote jobModel path =" + keyBuilder.getJobModelPath(jobModelVersion));
  } catch (Exception e) {
    LOG.error("JobModel publish failed for version=" + jobModelVersion, e);
    throw new SamzaException(e);
  }
}
 

import org.apache.samza.job.model.JobModel; //导入依赖的package包/类
/**
 * get the job model from ZK by version
 * @param jobModelVersion jobModel version to get
 * @return job model for this version
 */
public JobModel getJobModel(String jobModelVersion) {
  LOG.info("read the model ver=" + jobModelVersion + " from " + keyBuilder.getJobModelPath(jobModelVersion));
  Object data = zkClient.readData(keyBuilder.getJobModelPath(jobModelVersion));
  metrics.reads.inc();
  ObjectMapper mmapper = SamzaObjectMapper.getObjectMapper();
  JobModel jm;
  try {
    jm = mmapper.readValue((String) data, JobModel.class);
  } catch (IOException e) {
    throw new SamzaException("failed to read JobModel from ZK", e);
  }
  return jm;
}
 

import org.apache.samza.job.model.JobModel; //导入依赖的package包/类
void doOnProcessorChange(List<String> processors) {
  // if list of processors is empty - it means we are called from 'onBecomeLeader'
  // TODO: Handle empty currentProcessorIds.
  List<String> currentProcessorIds = getActualProcessorIds(processors);
  Set<String> uniqueProcessorIds = new HashSet<String>(currentProcessorIds);

  if (currentProcessorIds.size() != uniqueProcessorIds.size()) {
    LOG.info("Processors: {} has duplicates. Not generating job model.", currentProcessorIds);
    return;
  }

  // Generate the JobModel
  JobModel jobModel = generateNewJobModel(currentProcessorIds);
  if (!hasCreatedChangeLogStreams) {
    JobModelManager.createChangeLogStreams(new StorageConfig(config), jobModel.maxChangeLogStreamPartitions);
    hasCreatedChangeLogStreams = true;
  }
  // Assign the next version of JobModel
  String currentJMVersion = zkUtils.getJobModelVersion();
  String nextJMVersion;
  if (currentJMVersion == null) {
    nextJMVersion = "1";
  } else {
    nextJMVersion = Integer.toString(Integer.valueOf(currentJMVersion) + 1);
  }
  LOG.info("pid=" + processorId + "Generated new Job Model. Version = " + nextJMVersion);

  // Publish the new job model
  zkUtils.publishJobModel(nextJMVersion, jobModel);

  // Start the barrier for the job model update
  barrier.create(nextJMVersion, currentProcessorIds);

  // Notify all processors about the new JobModel by updating JobModel Version number
  zkUtils.publishJobModelVersion(currentJMVersion, nextJMVersion);

  LOG.info("pid=" + processorId + "Published new Job Model. Version = " + nextJMVersion);

  debounceTimer.scheduleAfterDebounceTime(ON_ZK_CLEANUP, 0, () -> zkUtils.cleanupZK(NUM_VERSIONS_TO_LEAVE));
}