Java WindowPane类代码示例

import org.apache.samza.operators.windows.WindowPane; //导入依赖的package包/类
@Override
public Collection<WindowPane<K, Object>> handleTimer(MessageCollector collector, TaskCoordinator coordinator) {
  LOG.trace("Processing timer.");
  List<WindowPane<K, Object>> results = new ArrayList<>();
  List<TriggerKey<K>> keys = triggerScheduler.runPendingCallbacks();

  for (TriggerKey<K> key : keys) {
    TriggerImplHandler triggerImplHandler = triggers.get(key);
    if (triggerImplHandler != null) {
      Optional<WindowPane<K, Object>> maybeTriggeredPane = triggerImplHandler.onTimer(key, collector, coordinator);
      maybeTriggeredPane.ifPresent(results::add);
    }
  }
  LOG.trace("Triggered panes: " + results.size());
  return results;
}
 

import org.apache.samza.operators.windows.WindowPane; //导入依赖的package包/类
public Optional<WindowPane<K, Object>> onMessage(TriggerKey<K> triggerKey, M message,
    MessageCollector collector, TaskCoordinator coordinator) {
  if (!isCancelled) {
    LOG.trace("Forwarding callbacks for {}", message);
    impl.onMessage(message, triggerScheduler);

    if (impl.shouldFire()) {
      // repeating trigger can trigger multiple times, So, clear the state to allow future triggerings.
      if (impl instanceof RepeatingTriggerImpl) {
        ((RepeatingTriggerImpl<M, K>) impl).clear();
      }
      return onTriggerFired(triggerKey, collector, coordinator);
    }
  }
  return Optional.empty();
}
 

import org.apache.samza.operators.windows.WindowPane; //导入依赖的package包/类
@Override
public void init(StreamGraph graph, Config config) {
  Supplier<Integer> initialValue = () -> 0;
  FoldLeftFunction<PageViewEvent, Integer> counter = (m, c) -> c == null ? 1 : c + 1;
  MessageStream<PageViewEvent> inputStream = graph.getInputStream("inputStream", new JsonSerdeV2<PageViewEvent>());
  OutputStream<Integer> outputStream = graph.getOutputStream("outputStream", new IntegerSerde());

  // create a tumbling window that outputs the number of message collected every 10 minutes.
  // also emit early results if either the number of messages collected reaches 30000, or if no new messages arrive
  // for 1 minute.
  inputStream
      .window(Windows.tumblingWindow(Duration.ofMinutes(10), initialValue, counter, new IntegerSerde())
          .setLateTrigger(Triggers.any(Triggers.count(30000),
              Triggers.timeSinceLastMessage(Duration.ofMinutes(1)))), "window")
      .map(WindowPane::getMessage)
      .sendTo(outputStream);
}
 

import org.apache.samza.operators.windows.WindowPane; //导入依赖的package包/类
@Test
public void testWindowWithRelaxedTypes() throws Exception {
  StreamGraphImpl mockGraph = mock(StreamGraphImpl.class);
  OperatorSpec mockOpSpec = mock(OperatorSpec.class);
  MessageStream<TestInputMessageEnvelope> inputStream = new MessageStreamImpl<>(mockGraph, mockOpSpec);

  Function<TestMessageEnvelope, String> keyExtractor = m -> m.getKey();
  FoldLeftFunction<TestMessageEnvelope, Integer> aggregator = (m, c) -> c + 1;
  Supplier<Integer> initialValue = () -> 0;

  // should compile since TestMessageEnvelope (input for functions) is base class of TestInputMessageEnvelope (M)
  Window<TestInputMessageEnvelope, String, Integer> window =
      Windows.keyedTumblingWindow(keyExtractor, Duration.ofHours(1), initialValue, aggregator,
          null, mock(Serde.class));
  MessageStream<WindowPane<String, Integer>> windowedStream = inputStream.window(window, "w1");

  ArgumentCaptor<OperatorSpec> registeredOpCaptor = ArgumentCaptor.forClass(OperatorSpec.class);
  verify(mockOpSpec).registerNextOperatorSpec(registeredOpCaptor.capture());
  OperatorSpec<?, TestMessageEnvelope> registeredOpSpec = registeredOpCaptor.getValue();

  assertTrue(registeredOpSpec instanceof WindowOperatorSpec);
  assertEquals(OpCode.WINDOW, registeredOpSpec.getOpCode());
  assertEquals(window, ((WindowOperatorSpec) registeredOpSpec).getWindow());
}
 

import org.apache.samza.operators.windows.WindowPane; //导入依赖的package包/类
@Test
public void testNonKeyedTumblingWindowsDiscardingMode() throws Exception {

  StreamApplication sgb = new TumblingWindowStreamApplication(AccumulationMode.DISCARDING,
      Duration.ofSeconds(1), Triggers.repeat(Triggers.count(1000)));
  List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();

  TestClock testClock = new TestClock();
  StreamOperatorTask task = new StreamOperatorTask(sgb, runner, testClock);
  task.init(config, taskContext);

  MessageCollector messageCollector =
      envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());
  Assert.assertEquals(windowPanes.size(), 0);

  integers.forEach(n -> task.process(new IntegerEnvelope(n), messageCollector, taskCoordinator));
  Assert.assertEquals(windowPanes.size(), 0);

  testClock.advanceTime(Duration.ofSeconds(1));
  Assert.assertEquals(windowPanes.size(), 0);

  task.window(messageCollector, taskCoordinator);
  Assert.assertEquals(windowPanes.size(), 1);
  Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 9);
}
 

import org.apache.samza.operators.windows.WindowPane; //导入依赖的package包/类
@Test
public void testTumblingAggregatingWindowsDiscardingMode() throws Exception {

  when(taskContext.getStore("jobName-jobId-window-w1"))
      .thenReturn(new TestInMemoryStore<>(new TimeSeriesKeySerde(new IntegerSerde()), new IntegerSerde()));

  StreamApplication sgb = new AggregateTumblingWindowStreamApplication(AccumulationMode.DISCARDING,
      Duration.ofSeconds(1), Triggers.repeat(Triggers.count(2)));
  List<WindowPane<Integer, Integer>> windowPanes = new ArrayList<>();

  TestClock testClock = new TestClock();
  StreamOperatorTask task = new StreamOperatorTask(sgb, runner, testClock);
  task.init(config, taskContext);
  MessageCollector messageCollector = envelope -> windowPanes.add((WindowPane<Integer, Integer>) envelope.getMessage());
  integers.forEach(n -> task.process(new IntegerEnvelope(n), messageCollector, taskCoordinator));
  testClock.advanceTime(Duration.ofSeconds(1));

  task.window(messageCollector, taskCoordinator);
  Assert.assertEquals(windowPanes.size(), 5);
  Assert.assertEquals(windowPanes.get(0).getMessage(), new Integer(2));
  Assert.assertEquals(windowPanes.get(1).getMessage(), new Integer(2));
  Assert.assertEquals(windowPanes.get(2).getMessage(), new Integer(2));
  Assert.assertEquals(windowPanes.get(3).getMessage(), new Integer(2));
  Assert.assertEquals(windowPanes.get(4).getMessage(), new Integer(1));
}
 

import org.apache.samza.operators.windows.WindowPane; //导入依赖的package包/类
@Override
protected Collection<WindowPane<K, Object>> handleEndOfStream(MessageCollector collector, TaskCoordinator coordinator) {
  List<WindowPane<K, Object>> results = new ArrayList<>();
  Set<TriggerKey<K>> triggerKeys = new HashSet<>(triggers.keySet());
  for(TriggerKey<K> triggerKey : triggerKeys) {
    Optional<WindowPane<K, Object>> triggerResult = onTriggerFired(triggerKey, collector, coordinator);
    triggerResult.ifPresent(results::add);
  }
  return results;
}
 

import org.apache.samza.operators.windows.WindowPane; //导入依赖的package包/类
/**
 * Computes the pane output corresponding to a {@link TriggerKey} that fired.
 */
private WindowPane<K, Object> computePaneOutput(TriggerKey<K> triggerKey, Object windowVal) {
  WindowKey<K> windowKey = new WindowKey(triggerKey.getKey(), Long.toString(triggerKey.getTimestamp()));
  WindowPane<K, Object> paneOutput =
      new WindowPane<>(windowKey, windowVal, window.getAccumulationMode(), triggerKey.getType());
  LOG.trace("Emitting pane output for trigger key {}", triggerKey);
  return paneOutput;
}
 

import org.apache.samza.operators.windows.WindowPane; //导入依赖的package包/类
public Optional<WindowPane<K, Object>> onTimer(TriggerKey<K> key, MessageCollector collector,
    TaskCoordinator coordinator) {
  if (impl.shouldFire() && !isCancelled) {
    LOG.trace("Triggering timer triggers");

    // repeating trigger can trigger multiple times, So, clear the trigger to allow future triggerings.
    if (impl instanceof RepeatingTriggerImpl) {
      ((RepeatingTriggerImpl<M, K>) impl).clear();
    }
    return onTriggerFired(key, collector, coordinator);
  }
  return Optional.empty();
}
 

import org.apache.samza.operators.windows.WindowPane; //导入依赖的package包/类
@Test
public void testTumblingWindowsDiscardingMode() throws Exception {

  StreamApplication sgb = new KeyedTumblingWindowStreamApplication(AccumulationMode.DISCARDING,
      Duration.ofSeconds(1), Triggers.repeat(Triggers.count(2)));
  List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();

  TestClock testClock = new TestClock();
  StreamOperatorTask task = new StreamOperatorTask(sgb, runner, testClock);
  task.init(config, taskContext);
  MessageCollector messageCollector =
      envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());
  integers.forEach(n -> task.process(new IntegerEnvelope(n), messageCollector, taskCoordinator));
  testClock.advanceTime(Duration.ofSeconds(1));

  task.window(messageCollector, taskCoordinator);
  Assert.assertEquals(windowPanes.size(), 5);
  Assert.assertEquals(windowPanes.get(0).getKey().getKey(), new Integer(1));
  Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 2);

  Assert.assertEquals(windowPanes.get(1).getKey().getKey(), new Integer(2));
  Assert.assertEquals((windowPanes.get(1).getMessage()).size(), 2);

  Assert.assertEquals(windowPanes.get(2).getKey().getKey(), new Integer(1));
  Assert.assertEquals((windowPanes.get(2).getMessage()).size(), 2);

  Assert.assertEquals(windowPanes.get(3).getKey().getKey(), new Integer(2));
  Assert.assertEquals((windowPanes.get(3).getMessage()).size(), 2);

  Assert.assertEquals(windowPanes.get(4).getKey().getKey(), new Integer(3));
  Assert.assertEquals((windowPanes.get(4).getMessage()).size(), 1);
}
 

import org.apache.samza.operators.windows.WindowPane; //导入依赖的package包/类
@Test
public void testTumblingWindowsAccumulatingMode() throws Exception {
  StreamApplication sgb = new KeyedTumblingWindowStreamApplication(AccumulationMode.ACCUMULATING,
      Duration.ofSeconds(1), Triggers.repeat(Triggers.count(2)));
  List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();
  TestClock testClock = new TestClock();
  StreamOperatorTask task = new StreamOperatorTask(sgb, runner, testClock);
  task.init(config, taskContext);

  MessageCollector messageCollector =
      envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());
  integers.forEach(n -> task.process(new IntegerEnvelope(n), messageCollector, taskCoordinator));
  testClock.advanceTime(Duration.ofSeconds(1));
  task.window(messageCollector, taskCoordinator);

  Assert.assertEquals(windowPanes.size(), 7);
  Assert.assertEquals(windowPanes.get(0).getKey().getKey(), new Integer(1));
  Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 2);

  Assert.assertEquals(windowPanes.get(1).getKey().getKey(), new Integer(2));
  Assert.assertEquals((windowPanes.get(1).getMessage()).size(), 2);

  Assert.assertEquals(windowPanes.get(2).getKey().getKey(), new Integer(1));
  Assert.assertEquals((windowPanes.get(2).getMessage()).size(), 4);

  Assert.assertEquals(windowPanes.get(3).getKey().getKey(), new Integer(2));
  Assert.assertEquals((windowPanes.get(3).getMessage()).size(), 4);
}
 

import org.apache.samza.operators.windows.WindowPane; //导入依赖的package包/类
@Test
public void testSessionWindowsAccumulatingMode() throws Exception {
  StreamApplication sgb = new KeyedSessionWindowStreamApplication(AccumulationMode.DISCARDING,
      Duration.ofMillis(500));
  TestClock testClock = new TestClock();
  StreamOperatorTask task = new StreamOperatorTask(sgb, runner, testClock);
  List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();

  MessageCollector messageCollector =
      envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());
  task.init(config, taskContext);

  task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator);
  task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator);
  testClock.advanceTime(Duration.ofSeconds(1));

  task.process(new IntegerEnvelope(2), messageCollector, taskCoordinator);
  task.process(new IntegerEnvelope(2), messageCollector, taskCoordinator);

  task.process(new IntegerEnvelope(2), messageCollector, taskCoordinator);
  task.process(new IntegerEnvelope(2), messageCollector, taskCoordinator);

  testClock.advanceTime(Duration.ofSeconds(1));
  task.window(messageCollector, taskCoordinator);
  Assert.assertEquals(windowPanes.size(), 2);
  Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 2);
  Assert.assertEquals(windowPanes.get(0).getKey().getKey(), new Integer(1));
  Assert.assertEquals(windowPanes.get(1).getKey().getKey(), new Integer(2));
  Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 2);
  Assert.assertEquals((windowPanes.get(1).getMessage()).size(), 4);
}
 

import org.apache.samza.operators.windows.WindowPane; //导入依赖的package包/类
@Test
public void testCancelationOfRepeatingNestedTriggers() throws Exception {

  StreamApplication sgb = new KeyedTumblingWindowStreamApplication(AccumulationMode.ACCUMULATING, Duration.ofSeconds(1),
      Triggers.repeat(Triggers.any(Triggers.count(2), Triggers.timeSinceFirstMessage(Duration.ofMillis(500)))));
  List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();

  MessageCollector messageCollector =
      envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());

  TestClock testClock = new TestClock();
  StreamOperatorTask task = new StreamOperatorTask(sgb, runner, testClock);
  task.init(config, taskContext);

  task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator);

  task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator);
  //assert that the count trigger fired
  Assert.assertEquals(windowPanes.size(), 1);

  //advance the timer to enable the potential triggering of the inner timeSinceFirstMessage trigger
  task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator);
  testClock.advanceTime(Duration.ofMillis(500));
  //assert that the triggering of the count trigger cancelled the inner timeSinceFirstMessage trigger
  task.window(messageCollector, taskCoordinator);
  Assert.assertEquals(windowPanes.size(), 2);

  task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator);
  task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator);
  Assert.assertEquals(windowPanes.size(), 3);

  task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator);
  //advance timer by 500 more millis to enable the default trigger
  testClock.advanceTime(Duration.ofMillis(500));
  task.window(messageCollector, taskCoordinator);
  //assert that the default trigger fired
  Assert.assertEquals(windowPanes.size(), 4);
}
 

import org.apache.samza.operators.windows.WindowPane; //导入依赖的package包/类
@Test
public void testEndOfStreamFlushesWithEarlyTriggerFirings() throws Exception {
  EndOfStreamStates endOfStreamStates = new EndOfStreamStates(ImmutableSet.of(new SystemStreamPartition("kafka",
      "integers", new Partition(0))), Collections.emptyMap());

  when(taskContext.getTaskName()).thenReturn(new TaskName("task 1"));
  when(taskContext.fetchObject(EndOfStreamStates.class.getName())).thenReturn(endOfStreamStates);
  when(taskContext.fetchObject(WatermarkStates.class.getName())).thenReturn(mock(WatermarkStates.class));

  StreamApplication sgb = new TumblingWindowStreamApplication(AccumulationMode.DISCARDING,
      Duration.ofSeconds(1), Triggers.repeat(Triggers.count(2)));
  List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();

  TestClock testClock = new TestClock();
  StreamOperatorTask task = new StreamOperatorTask(sgb, runner, testClock);
  task.init(config, taskContext);

  MessageCollector messageCollector =
      envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());
  Assert.assertEquals(windowPanes.size(), 0);

  List<Integer> integerList = ImmutableList.of(1, 2, 1, 2, 1);
  integerList.forEach(n -> task.process(new IntegerEnvelope(n), messageCollector, taskCoordinator));

  // early triggers should emit (1,2) and (1,2) in the same window.
  Assert.assertEquals(windowPanes.size(), 2);

  testClock.advanceTime(Duration.ofSeconds(1));
  Assert.assertEquals(windowPanes.size(), 2);

  final IncomingMessageEnvelope endOfStream = IncomingMessageEnvelope.buildEndOfStreamEnvelope(
      new SystemStreamPartition("kafka", "integers", new Partition(0)));
  task.process(endOfStream, messageCollector, taskCoordinator);

  // end of stream flushes the last entry (1)
  Assert.assertEquals(windowPanes.size(), 3);
  Assert.assertEquals((windowPanes.get(0).getMessage()).size(), 2);
  verify(taskCoordinator, times(1)).commit(TaskCoordinator.RequestScope.CURRENT_TASK);
  verify(taskCoordinator, times(1)).shutdown(TaskCoordinator.RequestScope.CURRENT_TASK);
}
 

import org.apache.samza.operators.windows.WindowPane; //导入依赖的package包/类
@Test
public void testEndOfStreamFlushesWithDefaultTriggerFirings() throws Exception {
  EndOfStreamStates endOfStreamStates = new EndOfStreamStates(ImmutableSet.of(new SystemStreamPartition("kafka",
      "integers", new Partition(0))), Collections.emptyMap());

  when(taskContext.getTaskName()).thenReturn(new TaskName("task 1"));
  when(taskContext.fetchObject(EndOfStreamStates.class.getName())).thenReturn(endOfStreamStates);
  when(taskContext.fetchObject(WatermarkStates.class.getName())).thenReturn(mock(WatermarkStates.class));

  StreamApplication sgb = new KeyedSessionWindowStreamApplication(AccumulationMode.DISCARDING, Duration.ofMillis(500));
  TestClock testClock = new TestClock();
  List<WindowPane<Integer, Collection<IntegerEnvelope>>> windowPanes = new ArrayList<>();
  StreamOperatorTask task = new StreamOperatorTask(sgb, runner, testClock);
  task.init(config, taskContext);

  MessageCollector messageCollector =
      envelope -> windowPanes.add((WindowPane<Integer, Collection<IntegerEnvelope>>) envelope.getMessage());

  task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator);
  task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator);
  testClock.advanceTime(1000);
  task.window(messageCollector, taskCoordinator);
  Assert.assertEquals(windowPanes.size(), 1);

  task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator);
  task.process(new IntegerEnvelope(1), messageCollector, taskCoordinator);

  final IncomingMessageEnvelope endOfStream = IncomingMessageEnvelope.buildEndOfStreamEnvelope(
      new SystemStreamPartition("kafka", "integers", new Partition(0)));
  task.process(endOfStream, messageCollector, taskCoordinator);
  Assert.assertEquals(windowPanes.size(), 2);
  Assert.assertEquals(windowPanes.get(0).getMessage().size(), 2);
  verify(taskCoordinator, times(1)).commit(TaskCoordinator.RequestScope.CURRENT_TASK);
  verify(taskCoordinator, times(1)).shutdown(TaskCoordinator.RequestScope.CURRENT_TASK);
}