Java Collector类代码示例

import org.apache.flink.util.Collector; //导入依赖的package包/类
public static void main(String... args) throws Exception {

        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        DataStream<WikipediaEditEvent> edits = env.addSource(new WikipediaEditsSource());

        edits
            .timeWindowAll(Time.minutes(1))
            .apply(new AllWindowFunction<WikipediaEditEvent, Tuple3<Date, Long, Long>, TimeWindow>() {
                @Override
                public void apply(TimeWindow timeWindow, Iterable<WikipediaEditEvent> iterable, Collector<Tuple3<Date, Long, Long>> collector) throws Exception {
                    long count = 0;
                    long bytesChanged = 0;

                    for (WikipediaEditEvent event : iterable) {
                        count++;
                        bytesChanged += event.getByteDiff();
                    }

                    collector.collect(new Tuple3<>(new Date(timeWindow.getEnd()), count, bytesChanged));
                }
            })
            .print();


        env.execute();
    }
 

import org.apache.flink.util.Collector; //导入依赖的package包/类
@Override
public void flatMap(String tweetJsonStr, Collector<Tuple2<String, Integer>> collector) throws Exception {
    JsonNode tweetJson = mapper.readTree(tweetJsonStr);
    JsonNode entities = tweetJson.get("entities");
    if (entities == null) return;

    JsonNode hashtags = entities.get("hashtags");
    if (hashtags == null) return;

    for (Iterator<JsonNode> iter = hashtags.getElements(); iter.hasNext();) {
        JsonNode node = iter.next();
        String hashtag = node.get("text").getTextValue();

        if (hashtag.matches("\\w+")) {
            collector.collect(new Tuple2<>(hashtag, 1));
        }
    }
}
 

import org.apache.flink.util.Collector; //导入依赖的package包/类
@Override
public void reduce(Iterable<Tuple4<Long, Integer, String, TagEvent>> values, Collector<ChartsResult> out) throws Exception {
    int counter = 0;
    String group= "";
    log.info("Reducing Groups and applyting limit(" + limit + ") by field " + groupPosition);

    for (Tuple4<Long, Integer, String, TagEvent> t : values) {
        if (!t.getField(groupPosition).equals(group))    {
            counter= 0;
            group= t.f2;
        }
        if (counter < limit) {
            out.collect(new ChartsResult(t.f0, t.f1, t.f3));
        }
        counter++;
    }
}
 

import org.apache.flink.util.Collector; //导入依赖的package包/类
/**
 * Evaluates the window and outputs none or several elements.
 *
 * @param window The window that is being evaluated.
 * @param values The elements in the window being evaluated.
 * @param out    A collector for emitting elements.
 * @throws Exception The function may throw exceptions to fail the program and trigger recovery.
 */
@Override
public void apply(TimeWindow window, Iterable<WindowWordWithCount> values, Collector<WindowWordRanking> out) throws Exception {
  this.ranking.setWStart(window.getStart());
  this.ranking.setWEnd(window.getEnd());

  List<WindowWordWithCount> rank = this.ranking.getRank();

  rank.clear();
  values.forEach(rank::add);
  rank.sort((e1,e2) -> Long.compare(e2.getCount(), e1.getCount()));
  int size = rank.size();
  rank.subList(Math.min(this.rankSize, size), size).clear();

  LOG.debug("WOUT: {}", this.ranking);

  out.collect(this.ranking);
}
 

import org.apache.flink.util.Collector; //导入依赖的package包/类
@Override
public void flatMap(RichSensorEvent event, Collector<PlayerSpeedStatistics> out) throws Exception {
  if (event.equals(this.eos)) {
    //LOG.debug("EOS RECEIVED");
    out.collect(new PlayerSpeedStatistics(0,0,0, this.averageSpeed));
    out.collect(END_OF_STREAM);
    super.close();
  }

  this.numEvents++;

  //LOG.debug("IN ({}): {}", this.numEvents, event);

  final double speed = PhysicsUtil.computeSpeed(event.getV(), event.getVx(), event.getVy(), event.getA(), event.getAx(), event.getAy());

  this.averageSpeed = ((this.averageSpeed * (this.numEvents - 1)) + speed) / this.numEvents;

  //LOG.debug("ACC: {} {}", this.numEvents, this.averageSpeed);
}
 

import org.apache.flink.util.Collector; //导入依赖的package包/类
/**
 * Evaluates the window and outputs none or several elements.
 *
 * @param window The window that is being evaluated.
 * @param values The elements in the window being evaluated.
 * @param out    A collector for emitting elements.
 * @throws Exception The function may throw exceptions to fail the program and trigger recovery.
 */
@Override
public void apply(TimeWindow window, Iterable<PlayerSpeedStatistics> values, Collector<PlayersSpeedRanking> out) throws Exception {
  this.ranking.setTsStart(window.getStart());
  this.ranking.setTsStop(window.getEnd());

  List<RankingElement> rank = this.ranking.getRank();

  rank.clear();
  values.forEach(e -> rank.add(new RankingElement(e.getPid(), e.getAverageSpeed())));
  rank.sort(Comparator.reverseOrder());
  int size = rank.size();
  rank.subList(Math.min(this.rankSize, size), size).clear();

  //LOG.debug("WOUT: {}", this.ranking);

  out.collect(this.ranking);
}
 

import org.apache.flink.util.Collector; //导入依赖的package包/类
@Override
public void processElement(Tuple3<String, Long, String> message, Context context,
    Collector<Tuple3<String, Long, String>> out) throws Exception {

  TimerService timerService = context.timerService();

  if (context.timestamp() > timerService.currentWatermark()) {
    PriorityQueue<Tuple3<String, Long, String>> queue = queueState.value();
    if (queue == null) {
      queue = new PriorityQueue<>(15, new RawMessageTuplesComparator());
    }
    queue.add(message);
    queueState.update(queue);
    // register a timer to be fired when the watermark passes this message timestamp
    timerService.registerEventTimeTimer(message.f1);
  } else {
    String outOfOrderErrorMessage = "out of order message: " + message.f2;
    logger.info(outOfOrderErrorMessage);
    throw new Exception(outOfOrderErrorMessage);
  }
}
 

import org.apache.flink.util.Collector; //导入依赖的package包/类
@Override
public void processElement(AisMessage message, Context context, Collector<AisMessage> out)
    throws Exception {

  TimerService timerService = context.timerService();

  PriorityQueue<AisMessage> queue = queueState.value();
  if (queue == null) {
    queue =
        new PriorityQueue<>(MAX_NUMBER_OF_QUEUED_ELEMENTS + 1, new PositionMessagesComparator());
  }
  long timestamp = System.currentTimeMillis();
  if (context.timestamp() > timerService.currentWatermark()) {
    queue.add(message);
    queueState.update(queue);
    // register a timer to be fired when the watermark passes this message timestamp
    timerService.registerEventTimeTimer(timestamp);
  } else {
    // logger.info("out of order message: " + message.toString());
    // throw new Exception(timerService.currentWatermark() + "out of order message: "
    // + message.toString());
    queue.add(message);
    queueState.update(queue);

  }
}
 

import org.apache.flink.util.Collector; //导入依赖的package包/类
@Override
public void flatMap(Event event, Collector<Event.Alert> collector) throws Exception {
	EventStateMachine.State value = state.value();
	if(value == null) {
		value = EventStateMachine.Transitions.initialState;
	}

	EventStateMachine.State nextValue = value.transition(event.getEvent());
	if(nextValue instanceof EventStateMachine.InvalidTransition) {
		Event.Alert alert = new Event.Alert(Instant.now(), event, value);
		collector.collect(alert);
		state.update(null);
	} else if (!nextValue.terminal()){
		state.update(nextValue);
	} else {
		state.update(null);
	}
}
 

import org.apache.flink.util.Collector; //导入依赖的package包/类
/**
 * @param key the key to calculate by (address or id in this section)
 * @param window the time window
 * @param lamps the Iterable<{@link StreetLamp}>
 * @param collector the collector to handle the hand off between streams
 * @throws Exception
 */
@Override
public void apply(Integer key, TimeWindow window, Iterable<StreetLamp> lamps, Collector<LampEMAConsumption> collector) throws Exception {

    // creating lamp consumption
    LampEMAConsumption lampConsumption = new LampEMAConsumption();
    lampConsumption.setId(key);
    lampConsumption.setComputed(Instant.now().getEpochSecond());

    // iterating over lamps in window
    for (Object lamp : lamps) {
        if (lampConsumption.getConsumption() == 0) {
            // initializing average
            lampConsumption.setConsumption(((StreetLamp) lamp).getConsumption());
            lampConsumption.setAddress(((StreetLamp) lamp).getAddress());
        } else {
            // using EMA to calculate average on the selected window
            // the formula is: S_t = alpha * Y_t + (1 - alpha) * S_t-1
            lampConsumption.setConsumption(ALPHA * ((StreetLamp) lamp).getConsumption() + (1 - ALPHA) * lampConsumption.getConsumption());
        }
    }

    // returning the consumption
    collector.collect(lampConsumption);
}
 

import org.apache.flink.util.Collector; //导入依赖的package包/类
/**
 * @param key the key to calculate by (address or id in this section)
 * @param window the time window
 * @param lamps the Iterable<{@link StreetLamp}>
 * @param collector the collector to handle the hand off between streams
 * @throws Exception
 */
@Override
public void apply(String key, TimeWindow window, Iterable<StreetLamp> lamps, Collector<LampEMAConsumptionStreet> collector) throws Exception {

    // creating lamp consumption
    LampEMAConsumptionStreet streetConsumption = new LampEMAConsumptionStreet();
    streetConsumption.setAddress(key);
    streetConsumption.setComputed(Instant.now().getEpochSecond());

    // iterating over lamps in window
    for (Object lamp : lamps) {
        if (streetConsumption.getConsumption() == 0) {
            // initializing average
            streetConsumption.setConsumption(((StreetLamp) lamp).getConsumption());
        } else {
            // using EMA to calculate average on the selected window
            // the formula is: S_t = alpha * Y_t + (1 - alpha) * S_t-1
            streetConsumption.setConsumption(ALPHA * ((StreetLamp) lamp).getConsumption() + (1 - ALPHA) * streetConsumption.getConsumption());
        }
    }

    // returning the consumption
    collector.collect(streetConsumption);
}
 

import org.apache.flink.util.Collector; //导入依赖的package包/类
public ReduceFacade(ReduceFunction<T> reducer, Collector<T> outputCollector, boolean objectReuseEnabled) {
	this.reducer = reducer;
	this.outputCollector = outputCollector;
	this.objectReuseEnabled = objectReuseEnabled;
	this.prober = getProber(buildSideComparator, new SameTypePairComparator<>(buildSideComparator));
	this.reuse = buildSideSerializer.createInstance();
}
 

import org.apache.flink.util.Collector; //导入依赖的package包/类
@Override
public void reduce(Iterable<Edge> edgesIter, Collector<Triad> out) throws Exception {

	final Iterator<Edge> edges = edgesIter.iterator();

	// clear vertex list
	vertices.clear();

	// read first edge
	Edge firstEdge = edges.next();
	outTriad.setFirstVertex(firstEdge.getFirstVertex());
	vertices.add(firstEdge.getSecondVertex());

	// build and emit triads
	while (edges.hasNext()) {
		Integer higherVertexId = edges.next().getSecondVertex();

		// combine vertex with all previously read vertices
		for (Integer lowerVertexId : vertices) {
			outTriad.setSecondVertex(lowerVertexId);
			outTriad.setThirdVertex(higherVertexId);
			out.collect(outTriad);
		}
		vertices.add(higherVertexId);
	}
}
 

import org.apache.flink.util.Collector; //导入依赖的package包/类
@Override
public void combine(Iterable<Tuple3<Integer, Long, String>> values, Collector<Tuple3<Integer, Long, String>> out) {

	Iterator<Tuple3<Integer, Long, String>> it = values.iterator();
	Tuple3<Integer, Long, String> t = it.next();

	int i = t.f0;
	out.collect(t);

	while (it.hasNext()) {
		t = it.next();
		if (i > t.f0) {
			t.f2 = "INVALID-ORDER!";
			out.collect(t);
		}
	}

}
 

import org.apache.flink.util.Collector; //导入依赖的package包/类
@Override
public void coGroup(Iterable<Edge<K, EV>> edges, Iterable<Vertex<K, Tuple3<VV, LongValue, LongValue>>> state,
					Collector<Tuple2<K, Message>> out) throws Exception {

	final Iterator<Vertex<K, Tuple3<VV, LongValue, LongValue>>> stateIter = state.iterator();

	if (stateIter.hasNext()) {
		Vertex<K, Tuple3<VV, LongValue, LongValue>> vertexWithDegrees = stateIter.next();

		nextVertex.f0 = vertexWithDegrees.f0;
		nextVertex.f1 = vertexWithDegrees.f1.f0;

		scatterFunction.setInDegree(vertexWithDegrees.f1.f1.getValue());
		scatterFunction.setOutDegree(vertexWithDegrees.f1.f2.getValue());

		scatterFunction.set(edges.iterator(), out, vertexWithDegrees.getId());
		scatterFunction.sendMessages(nextVertex);
	}
}