Java KStreamBuilder.build方法代码示例

import org.apache.kafka.streams.kstream.KStreamBuilder; //导入方法依赖的package包/类
public KStreamTestDriver(final KStreamBuilder builder,
                         final File stateDir,
                         final Serde<?> keySerde,
                         final Serde<?> valSerde,
                         final long cacheSize) {
    builder.setApplicationId("TestDriver");
    topology = builder.build(null);
    globalTopology = builder.buildGlobalStateTopology();
    final ThreadCache cache = new ThreadCache("testCache", cacheSize, new MockStreamsMetrics(new Metrics()));
    context = new MockProcessorContext(stateDir, keySerde, valSerde, new MockRecordCollector(), cache);
    context.setRecordContext(new ProcessorRecordContext(0, 0, 0, "topic"));
    // init global topology first as it will add stores to the
    // store map that are required for joins etc.
    if (globalTopology != null) {
        initTopology(globalTopology, globalTopology.globalStateStores());
    }
    initTopology(topology, topology.stateStores());
}
 

import org.apache.kafka.streams.kstream.KStreamBuilder; //导入方法依赖的package包/类
@Test
public void shouldCloseActiveTasksThatAreAssignedToThisStreamThreadButAssignmentHasChangedBeforeCreatingNewTasks() throws Exception {
    final KStreamBuilder builder = new KStreamBuilder();
    builder.setApplicationId(applicationId);
    builder.stream(Pattern.compile("t.*")).to("out");

    final Map<Collection<TopicPartition>, TestStreamTask> createdTasks = new HashMap<>();

    final StreamThread thread = new StreamThread(
        builder,
        config,
        clientSupplier,
        applicationId,
        clientId,
        processId,
        metrics,
        mockTime,
        new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST),
        0) {

        @Override
        protected StreamTask createStreamTask(final TaskId id, final Collection<TopicPartition> partitions) {
            final ProcessorTopology topology = builder.build(id.topicGroupId);
            final TestStreamTask task = new TestStreamTask(
                id,
                applicationId,
                partitions,
                topology,
                consumer,
                clientSupplier.getProducer(new HashMap<String, Object>()),
                restoreConsumer,
                config,
                new MockStreamsMetrics(new Metrics()),
                stateDirectory);
            createdTasks.put(partitions, task);
            return task;
        }
    };

    final Map<TaskId, Set<TopicPartition>> activeTasks = new HashMap<>();
    final TopicPartition t1 = new TopicPartition("t1", 0);
    final Set<TopicPartition> task00Partitions = new HashSet<>();
    task00Partitions.add(t1);
    final TaskId taskId = new TaskId(0, 0);
    activeTasks.put(taskId, task00Partitions);

    thread.setPartitionAssignor(new StreamPartitionAssignor() {
        @Override
        Map<TaskId, Set<TopicPartition>> activeTasks() {
            return activeTasks;
        }
    });

    StreamPartitionAssignor.SubscriptionUpdates subscriptionUpdates = new StreamPartitionAssignor.SubscriptionUpdates();
    Field updatedTopicsField  = subscriptionUpdates.getClass().getDeclaredField("updatedTopicSubscriptions");
    updatedTopicsField.setAccessible(true);
    Set<String> updatedTopics = (Set<String>) updatedTopicsField.get(subscriptionUpdates);
    updatedTopics.add(t1.topic());
    builder.updateSubscriptions(subscriptionUpdates, null);

    // should create task for id 0_0 with a single partition
    thread.rebalanceListener.onPartitionsRevoked(Collections.<TopicPartition>emptyList());
    thread.rebalanceListener.onPartitionsAssigned(task00Partitions);

    final TestStreamTask firstTask = createdTasks.get(task00Partitions);
    assertThat(firstTask.id(), is(taskId));

    // update assignment for the task 0_0 so it now has 2 partitions
    task00Partitions.add(new TopicPartition("t2", 0));
    updatedTopics.add("t2");

    thread.rebalanceListener.onPartitionsRevoked(Collections.<TopicPartition>emptyList());
    thread.rebalanceListener.onPartitionsAssigned(task00Partitions);

    // should close the first task as the assignment has changed
    assertTrue("task should have been closed as assignment has changed", firstTask.closed);
    assertTrue("tasks state manager should have been closed as assignment has changed", firstTask.closedStateManager);
    // should have created a new task for 00
    assertThat(createdTasks.get(task00Partitions).id(), is(taskId));
}
 

import org.apache.kafka.streams.kstream.KStreamBuilder; //导入方法依赖的package包/类
@Test
public void shouldNotViolateAtLeastOnceWhenAnExceptionOccursOnTaskCloseDuringShutdown() throws Exception {
    final KStreamBuilder builder = new KStreamBuilder();
    builder.setApplicationId(applicationId);
    builder.stream("t1").groupByKey();

    final TestStreamTask testStreamTask = new TestStreamTask(
        new TaskId(0, 0),
        applicationId,
        Utils.mkSet(new TopicPartition("t1", 0)),
        builder.build(0),
        clientSupplier.consumer,
        clientSupplier.getProducer(new HashMap<String, Object>()),
        clientSupplier.restoreConsumer,
        config,
        new MockStreamsMetrics(new Metrics()),
        new StateDirectory(applicationId, config.getString(StreamsConfig.STATE_DIR_CONFIG), mockTime)) {

        @Override
        public void close(final boolean clean) {
            throw new RuntimeException("KABOOM!");
        }
    };

    final StreamThread thread = new StreamThread(
        builder,
        config,
        clientSupplier,
        applicationId,
        clientId,
        processId,
        metrics,
        mockTime,
        new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST),
        0) {

        @Override
        protected StreamTask createStreamTask(final TaskId id, final Collection<TopicPartition> partitions) {
            return testStreamTask;
        }
    };

    final Map<TaskId, Set<TopicPartition>> activeTasks = new HashMap<>();
    activeTasks.put(testStreamTask.id(), testStreamTask.partitions);

    thread.setPartitionAssignor(new MockStreamsPartitionAssignor(activeTasks));

    thread.rebalanceListener.onPartitionsRevoked(Collections.<TopicPartition>emptyList());
    thread.rebalanceListener.onPartitionsAssigned(testStreamTask.partitions);

    thread.start();
    thread.close();
    thread.join();
    assertFalse("task shouldn't have been committed as there was an exception during shutdown", testStreamTask.committed);
}
 

import org.apache.kafka.streams.kstream.KStreamBuilder; //导入方法依赖的package包/类
@Test
public void shouldNotViolateAtLeastOnceWhenExceptionOccursDuringTaskSuspension() throws Exception {
    final KStreamBuilder builder = new KStreamBuilder();
    builder.setApplicationId(applicationId);
    builder.stream("t1").groupByKey();

    final TestStreamTask testStreamTask = new TestStreamTask(
        new TaskId(0, 0),
        applicationId,
        Utils.mkSet(new TopicPartition("t1", 0)),
        builder.build(0),
        clientSupplier.consumer,
        clientSupplier.getProducer(new HashMap<String, Object>()),
        clientSupplier.restoreConsumer,
        config,
        new MockStreamsMetrics(new Metrics()),
        new StateDirectory(applicationId, config.getString(StreamsConfig.STATE_DIR_CONFIG), mockTime)) {

        @Override
        public void suspend() {
            throw new RuntimeException("KABOOM!");
        }
    };

    final StreamThread thread = new StreamThread(
        builder,
        config,
        clientSupplier,
        applicationId,
        clientId,
        processId,
        metrics,
        mockTime,
        new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST),
        0) {

        @Override
        protected StreamTask createStreamTask(final TaskId id, final Collection<TopicPartition> partitions) {
            return testStreamTask;
        }
    };


    final Map<TaskId, Set<TopicPartition>> activeTasks = new HashMap<>();
    activeTasks.put(testStreamTask.id(), testStreamTask.partitions);


    thread.setPartitionAssignor(new MockStreamsPartitionAssignor(activeTasks));

    thread.rebalanceListener.onPartitionsRevoked(Collections.<TopicPartition>emptyList());
    thread.rebalanceListener.onPartitionsAssigned(testStreamTask.partitions);
    try {
        thread.rebalanceListener.onPartitionsRevoked(Collections.<TopicPartition>emptyList());
        fail("should have thrown exception");
    } catch (final Exception e) {
        // expected
    }
    assertFalse(testStreamTask.committed);
}
 

import org.apache.kafka.streams.kstream.KStreamBuilder; //导入方法依赖的package包/类
@Test
public void shouldNotViolateAtLeastOnceWhenExceptionOccursDuringFlushStateWhileSuspendingState() throws Exception {
    final KStreamBuilder builder = new KStreamBuilder();
    builder.setApplicationId(applicationId);
    builder.stream("t1").groupByKey();

    final TestStreamTask testStreamTask = new TestStreamTask(
        new TaskId(0, 0),
        applicationId,
        Utils.mkSet(new TopicPartition("t1", 0)),
        builder.build(0),
        clientSupplier.consumer,
        clientSupplier.getProducer(new HashMap<String, Object>()),
        clientSupplier.restoreConsumer,
        config,
        new MockStreamsMetrics(new Metrics()),
        new StateDirectory(applicationId, config.getString(StreamsConfig.STATE_DIR_CONFIG), mockTime)) {

        @Override
        protected void flushState() {
            throw new RuntimeException("KABOOM!");
        }
    };

    final StreamThread thread = new StreamThread(
        builder,
        config,
        clientSupplier,
        applicationId,
        clientId,
        processId,
        metrics,
        mockTime,
        new StreamsMetadataState(builder, StreamsMetadataState.UNKNOWN_HOST), 0) {

        @Override
        protected StreamTask createStreamTask(final TaskId id, final Collection<TopicPartition> partitions) {
            return testStreamTask;
        }
    };


    final Map<TaskId, Set<TopicPartition>> activeTasks = new HashMap<>();
    activeTasks.put(testStreamTask.id(), testStreamTask.partitions);


    thread.setPartitionAssignor(new MockStreamsPartitionAssignor(activeTasks));

    thread.rebalanceListener.onPartitionsRevoked(Collections.<TopicPartition>emptyList());
    thread.rebalanceListener.onPartitionsAssigned(testStreamTask.partitions);
    try {
        thread.rebalanceListener.onPartitionsRevoked(Collections.<TopicPartition>emptyList());
        fail("should have thrown exception");
    } catch (final Exception e) {
        // expected
    }
    assertFalse(testStreamTask.committed);
}