Java StreamExecutionEnvironment.createLocalEnvironment方法代码示例

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();

    Properties properties = new Properties();
    properties.load(new FileInputStream("src/main/resources/application.properties"));

    Properties mqttProperties = new Properties();

    // client id = a:<Organization_ID>:<App_Id>
    mqttProperties.setProperty(MQTTSource.CLIENT_ID,
            String.format("a:%s:%s",
                    properties.getProperty("Org_ID"),
                    properties.getProperty("App_Id")));

    // mqtt server url = tcp://<Org_ID>.messaging.internetofthings.ibmcloud.com:1883
    mqttProperties.setProperty(MQTTSource.URL,
            String.format("tcp://%s.messaging.internetofthings.ibmcloud.com:1883",
                    properties.getProperty("Org_ID")));

    // topic = iot-2/type/<Device_Type>/id/<Device_ID>/evt/<Event_Id>/fmt/json
    mqttProperties.setProperty(MQTTSource.TOPIC,
            String.format("iot-2/type/%s/id/%s/evt/%s/fmt/json",
                    properties.getProperty("Device_Type"),
                    properties.getProperty("Device_ID"),
                    properties.getProperty("EVENT_ID")));

    mqttProperties.setProperty(MQTTSource.USERNAME, properties.getProperty("API_Key"));
    mqttProperties.setProperty(MQTTSource.PASSWORD, properties.getProperty("APP_Authentication_Token"));


    MQTTSource mqttSource = new MQTTSource(mqttProperties);
    DataStreamSource<String> tempratureDataSource = env.addSource(mqttSource);
    DataStream<String> stream = tempratureDataSource.map((MapFunction<String, String>) s -> s);
    stream.print();

    env.execute("Temperature Analysis");
}
 

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
public static void main(String[] args) throws IOException {
  StreamExecutionEnvironment execEnv = StreamExecutionEnvironment.createLocalEnvironment();
  StreamTableEnvironment env = StreamTableEnvironment.getTableEnvironment(execEnv);
  execEnv.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
  CompilationResult res = new CompilationResult();

  try {
    JobDescriptor job = getJobConf(System.in);
    res.jobGraph(new JobCompiler(env, job).getJobGraph());
  } catch (Throwable e) {
    res.remoteThrowable(e);
  }

  try (OutputStream out = chooseOutputStream(args)) {
    out.write(res.serialize());
  }
}
 

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
@Test
public void testEventTimeOrderedWriter() throws Exception {
    StreamExecutionEnvironment execEnv = StreamExecutionEnvironment.createLocalEnvironment();

    String streamName = "testEventTimeOrderedWriter";
    SETUP_UTILS.createTestStream(streamName, 1);

    DataStreamSource<Integer> dataStream = execEnv
            .addSource(new IntegerGeneratingSource(false, EVENT_COUNT_PER_SOURCE));

    FlinkPravegaWriter<Integer> pravegaSink = new FlinkPravegaWriter<>(
            SETUP_UTILS.getControllerUri(),
            SETUP_UTILS.getScope(),
            streamName,
            new IntSerializer(),
            event -> "fixedkey");

    FlinkPravegaUtils.writeToPravegaInEventTimeOrder(dataStream, pravegaSink, 1);
    Assert.assertNotNull(execEnv.getExecutionPlan());
}
 

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
private JobGraph createJobGraph(ExecutionMode mode) {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.enableCheckpointing(500, CheckpointingMode.EXACTLY_ONCE);
	env.setRestartStrategy(RestartStrategies.noRestart());
	env.setStateBackend(new MemoryStateBackend());

	switch (mode) {
		case MIGRATE:
			createMigrationJob(env);
			break;
		case RESTORE:
			createRestoredJob(env);
			break;
	}

	return StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
}
 

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
/**
 * Tests that there are no collisions with two identical sources.
 *
 * <pre>
 * [ (src0) ] --\
 *               +--> [ (sink) ]
 * [ (src1) ] --/
 * </pre>
 */
@Test
public void testNodeHashIdenticalSources() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);
	env.disableOperatorChaining();

	DataStream<String> src0 = env.addSource(new NoOpSourceFunction());
	DataStream<String> src1 = env.addSource(new NoOpSourceFunction());

	src0.union(src1).addSink(new NoOpSinkFunction());

	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	List<JobVertex> vertices = jobGraph.getVerticesSortedTopologicallyFromSources();
	assertTrue(vertices.get(0).isInputVertex());
	assertTrue(vertices.get(1).isInputVertex());

	assertNotNull(vertices.get(0).getID());
	assertNotNull(vertices.get(1).getID());

	assertNotEquals(vertices.get(0).getID(), vertices.get(1).getID());
}
 

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
/**
 * Tests that there are no collisions with two identical intermediate nodes connected to the
 * same predecessor.
 *
 * <pre>
 *             /-> [ (map) ] -> [ (sink) ]
 * [ (src) ] -+
 *             \-> [ (map) ] -> [ (sink) ]
 * </pre>
 */
@Test
public void testNodeHashIdenticalNodes() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);
	env.disableOperatorChaining();

	DataStream<String> src = env.addSource(new NoOpSourceFunction());

	src.map(new NoOpMapFunction()).addSink(new NoOpSinkFunction());

	src.map(new NoOpMapFunction()).addSink(new NoOpSinkFunction());

	JobGraph jobGraph = env.getStreamGraph().getJobGraph();
	Set<JobVertexID> vertexIds = new HashSet<>();
	for (JobVertex vertex : jobGraph.getVertices()) {
		assertTrue(vertexIds.add(vertex.getID()));
	}
}
 

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
/**
 * Tests that a changed operator name does not affect the hash.
 */
@Test
public void testChangedOperatorName() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.addSource(new NoOpSourceFunction(), "A").map(new NoOpMapFunction());
	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	JobVertexID expected = jobGraph.getVerticesAsArray()[0].getID();

	env = StreamExecutionEnvironment.createLocalEnvironment();
	env.addSource(new NoOpSourceFunction(), "B").map(new NoOpMapFunction());
	jobGraph = env.getStreamGraph().getJobGraph();

	JobVertexID actual = jobGraph.getVerticesAsArray()[0].getID();

	assertEquals(expected, actual);
}
 

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
@Test
public void testUserProvidedHashing() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();

	List<String> userHashes = Arrays.asList("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb");

	env.addSource(new NoOpSourceFunction(), "src").setUidHash(userHashes.get(0))
			.map(new NoOpMapFunction())
			.filter(new NoOpFilterFunction())
			.keyBy(new NoOpKeySelector())
			.reduce(new NoOpReduceFunction()).name("reduce").setUidHash(userHashes.get(1));

	StreamGraph streamGraph = env.getStreamGraph();
	int idx = 1;
	for (JobVertex jobVertex : streamGraph.getJobGraph().getVertices()) {
		List<JobVertexID> idAlternatives = jobVertex.getIdAlternatives();
		Assert.assertEquals(idAlternatives.get(idAlternatives.size() - 1).toString(), userHashes.get(idx));
		--idx;
	}
}
 

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
public StreamExecutionEnvironment createExecutionEnvironment() {
    Boolean runLocally = parameters.getBoolean("run-locally", false);

    StreamExecutionEnvironment environment = runLocally
            ? StreamExecutionEnvironment.createLocalEnvironment()
            : StreamExecutionEnvironment.getExecutionEnvironment();

    environment.enableCheckpointing(parameters.getLong("flink-checkpoint-interval", 10000));
    environment.getCheckpointConfig().setCheckpointingMode(CheckpointingMode.EXACTLY_ONCE);
    environment.getCheckpointConfig().setMaxConcurrentCheckpoints(1);
    environment.getCheckpointConfig().setCheckpointTimeout(parameters.getLong("flink-checkpoint-timeout", 60000));

    return environment;
}
 

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
/**
 * Tests that (un)chaining affects the node hash (for sources).
 *
 * <pre>
 * A (chained): [ (src0) -> (map) -> (filter) -> (sink) ]
 * B (unchained): [ (src0) ] -> [ (map) -> (filter) -> (sink) ]
 * </pre>
 *
 * <p>The hashes for the single vertex in A and the source vertex in B need to be different.
 */
@Test
public void testNodeHashAfterSourceUnchaining() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);

	env.addSource(new NoOpSourceFunction())
			.map(new NoOpMapFunction())
			.filter(new NoOpFilterFunction())
			.addSink(new NoOpSinkFunction());

	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	JobVertexID sourceId = jobGraph.getVerticesSortedTopologicallyFromSources()
			.get(0).getID();

	env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);

	env.addSource(new NoOpSourceFunction())
			.map(new NoOpMapFunction())
			.startNewChain()
			.filter(new NoOpFilterFunction())
			.addSink(new NoOpSinkFunction());

	jobGraph = env.getStreamGraph().getJobGraph();

	JobVertexID unchainedSourceId = jobGraph.getVerticesSortedTopologicallyFromSources()
			.get(0).getID();

	assertNotEquals(sourceId, unchainedSourceId);
}
 

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
/**
 * Tests that (un)chaining affects the node hash (for intermediate nodes).
 *
 * <pre>
 * A (chained): [ (src0) -> (map) -> (filter) -> (sink) ]
 * B (unchained): [ (src0) ] -> [ (map) -> (filter) -> (sink) ]
 * </pre>
 *
 * <p>The hashes for the single vertex in A and the source vertex in B need to be different.
 */
@Test
public void testNodeHashAfterIntermediateUnchaining() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);

	env.addSource(new NoOpSourceFunction())
			.map(new NoOpMapFunction()).name("map")
			.startNewChain()
			.filter(new NoOpFilterFunction())
			.addSink(new NoOpSinkFunction());

	JobGraph jobGraph = env.getStreamGraph().getJobGraph();

	JobVertex chainedMap = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
	assertTrue(chainedMap.getName().startsWith("map"));
	JobVertexID chainedMapId = chainedMap.getID();

	env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);

	env.addSource(new NoOpSourceFunction())
			.map(new NoOpMapFunction()).name("map")
			.startNewChain()
			.filter(new NoOpFilterFunction())
			.startNewChain()
			.addSink(new NoOpSinkFunction());

	jobGraph = env.getStreamGraph().getJobGraph();

	JobVertex unchainedMap = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
	assertEquals("map", unchainedMap.getName());
	JobVertexID unchainedMapId = unchainedMap.getID();

	assertNotEquals(chainedMapId, unchainedMapId);
}
 

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
/**
 * Tests that a collision on the manual hash throws an Exception.
 */
@Test(expected = IllegalArgumentException.class)
public void testManualHashAssignmentCollisionThrowsException() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);
	env.disableOperatorChaining();

	env.addSource(new NoOpSourceFunction()).uid("source")
			.map(new NoOpMapFunction()).uid("source") // Collision
			.addSink(new NoOpSinkFunction());

	// This call is necessary to generate the job graph
	env.getStreamGraph().getJobGraph();
}
 

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
/**
 * Tests that a manual hash for an intermediate chain node is accepted.
 */
@Test
public void testManualHashAssignmentForIntermediateNodeInChain() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);

	env.addSource(new NoOpSourceFunction())
			// Intermediate chained node
			.map(new NoOpMapFunction()).uid("map")
			.addSink(new NoOpSinkFunction());

	env.getStreamGraph().getJobGraph();
}
 

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
/**
 * Tests that a manual hash at the beginning of a chain is accepted.
 */
@Test
public void testManualHashAssignmentForStartNodeInInChain() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(4);

	env.addSource(new NoOpSourceFunction()).uid("source")
			.map(new NoOpMapFunction())
			.addSink(new NoOpSinkFunction());

	env.getStreamGraph().getJobGraph();
}
 

import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; //导入方法依赖的package包/类
@Test
public void testUserProvidedHashingOnChainSupported() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();

	env.addSource(new NoOpSourceFunction(), "src").setUidHash("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
			.map(new NoOpMapFunction()).setUidHash("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
			.filter(new NoOpFilterFunction()).setUidHash("cccccccccccccccccccccccccccccccc")
			.keyBy(new NoOpKeySelector())
			.reduce(new NoOpReduceFunction()).name("reduce").setUidHash("dddddddddddddddddddddddddddddddd");

	env.getStreamGraph().getJobGraph();
}