Java TestStreamEnvironment类代码示例

import org.apache.flink.streaming.util.TestStreamEnvironment; //导入依赖的package包/类
@Override
public void after() {

	TestStreamEnvironment.unsetAsContext();
	TestEnvironment.unsetAsContext();

	final CompletableFuture<?> terminationFuture = jobExecutorService.terminate();

	try {
		terminationFuture.get(
			miniClusterResourceConfiguration.getShutdownTimeout().toMilliseconds(),
			TimeUnit.MILLISECONDS);
	} catch (Exception e) {
		LOG.warn("Could not properly shut down the MiniClusterResource.", e);
	}

	jobExecutorService = null;
}
 

import org.apache.flink.streaming.util.TestStreamEnvironment; //导入依赖的package包/类
@Test
public void testCheckpointedStreamingClassloaderJobWithCustomClassLoader() throws IOException, ProgramInvocationException {
	// checkpointed streaming job with custom classes for the checkpoint (FLINK-2543)
	// the test also ensures that user specific exceptions are serializable between JobManager <--> JobClient.
	PackagedProgram streamingCheckpointedProg = new PackagedProgram(new File(STREAMING_CHECKPOINTED_PROG_JAR_FILE));

	TestStreamEnvironment.setAsContext(
		testCluster,
		parallelism,
		Collections.singleton(new Path(STREAMING_CHECKPOINTED_PROG_JAR_FILE)),
		Collections.<URL>emptyList());

	// Program should terminate with a 'SuccessException':
	// we can not access the SuccessException here when executing the tests with maven, because its not available in the jar.
	expectedException.expectCause(
		Matchers.<Throwable>hasProperty("cause",
			hasProperty("class",
				hasProperty("canonicalName", equalTo(
					"org.apache.flink.test.classloading.jar.CheckpointedStreamingProgram.SuccessException")))));

	streamingCheckpointedProg.invokeInteractiveModeForExecution();
}
 

import org.apache.flink.streaming.util.TestStreamEnvironment; //导入依赖的package包/类
@Test
public void testCheckpointingCustomKvStateJobWithCustomClassLoader() throws IOException, ProgramInvocationException {
	File checkpointDir = FOLDER.newFolder();
	File outputDir = FOLDER.newFolder();

	final PackagedProgram program = new PackagedProgram(
		new File(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH),
		new String[] {
			checkpointDir.toURI().toString(),
			outputDir.toURI().toString()
		});

	TestStreamEnvironment.setAsContext(
		testCluster,
		parallelism,
		Collections.singleton(new Path(CHECKPOINTING_CUSTOM_KV_STATE_JAR_PATH)),
		Collections.<URL>emptyList());

	expectedException.expectCause(
		Matchers.<Throwable>hasProperty("cause", isA(SuccessException.class)));

	program.invokeInteractiveModeForExecution();
}
 

import org.apache.flink.streaming.util.TestStreamEnvironment; //导入依赖的package包/类
/**
 * Runs the following program the test program defined in {@link #testProgram(StreamExecutionEnvironment)}
 * followed by the checks in {@link #postSubmit}.
 */
@Test
public void runCheckpointedProgram() throws Exception {
	try {
		TestStreamEnvironment env = new TestStreamEnvironment(cluster, PARALLELISM);
		env.setParallelism(PARALLELISM);
		env.enableCheckpointing(500);
		env.getConfig().disableSysoutLogging();
		env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 0L));

		testProgram(env);

		TestUtils.tryExecute(env, "Fault Tolerance Test");

		postSubmit();
	}
	catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 

import org.apache.flink.streaming.util.TestStreamEnvironment; //导入依赖的package包/类
public static void prepare(boolean hideKafkaBehindProxy) throws ClassNotFoundException {
	LOG.info("-------------------------------------------------------------------------");
	LOG.info("    Starting KafkaTestBase ");
	LOG.info("-------------------------------------------------------------------------");

	startClusters(false, hideKafkaBehindProxy);

	TestStreamEnvironment.setAsContext(flink, PARALLELISM);
}
 

import org.apache.flink.streaming.util.TestStreamEnvironment; //导入依赖的package包/类
@AfterClass
public static void shutDownServices() throws Exception {

	LOG.info("-------------------------------------------------------------------------");
	LOG.info("    Shut down KafkaTestBase ");
	LOG.info("-------------------------------------------------------------------------");

	TestStreamEnvironment.unsetAsContext();

	shutdownClusters();

	LOG.info("-------------------------------------------------------------------------");
	LOG.info("    KafkaTestBase finished");
	LOG.info("-------------------------------------------------------------------------");
}
 

import org.apache.flink.streaming.util.TestStreamEnvironment; //导入依赖的package包/类
@BeforeClass
public static void prepare() throws IOException, ClassNotFoundException {
	LOG.info("-------------------------------------------------------------------------");
	LOG.info("    Starting KafkaShortRetentionTestBase ");
	LOG.info("-------------------------------------------------------------------------");

	Configuration flinkConfig = new Configuration();

	// dynamically load the implementation for the test
	Class<?> clazz = Class.forName("org.apache.flink.streaming.connectors.kafka.KafkaTestEnvironmentImpl");
	kafkaServer = (KafkaTestEnvironment) InstantiationUtil.instantiate(clazz);

	LOG.info("Starting KafkaTestBase.prepare() for Kafka " + kafkaServer.getVersion());

	if (kafkaServer.isSecureRunSupported()) {
		secureProps = kafkaServer.getSecureProperties();
	}

	Properties specificProperties = new Properties();
	specificProperties.setProperty("log.retention.hours", "0");
	specificProperties.setProperty("log.retention.minutes", "0");
	specificProperties.setProperty("log.retention.ms", "250");
	specificProperties.setProperty("log.retention.check.interval.ms", "100");
	kafkaServer.prepare(kafkaServer.createConfig().setKafkaServerProperties(specificProperties));

	standardProps = kafkaServer.getStandardProperties();

	// start also a re-usable Flink mini cluster
	flinkConfig.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
	flinkConfig.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, TM_SLOTS);
	flinkConfig.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 16L);
	flinkConfig.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s");

	flink = new LocalFlinkMiniCluster(flinkConfig, false);
	flink.start();

	TestStreamEnvironment.setAsContext(flink, PARALLELISM);
}
 

import org.apache.flink.streaming.util.TestStreamEnvironment; //导入依赖的package包/类
@AfterClass
public static void shutDownServices() throws Exception {
	TestStreamEnvironment.unsetAsContext();

	if (flink != null) {
		flink.stop();
	}
	kafkaServer.shutdown();

	secureProps.clear();
}
 

import org.apache.flink.streaming.util.TestStreamEnvironment; //导入依赖的package包/类
@Override
public void before() throws Exception {

	jobExecutorService = startJobExecutorService(miniClusterType);

	numberSlots = miniClusterResourceConfiguration.getNumberSlotsPerTaskManager() * miniClusterResourceConfiguration.getNumberTaskManagers();

	executionEnvironment = new TestEnvironment(jobExecutorService, numberSlots, false);
	executionEnvironment.setAsContext();
	TestStreamEnvironment.setAsContext(jobExecutorService, numberSlots);
}
 

import org.apache.flink.streaming.util.TestStreamEnvironment; //导入依赖的package包/类
@AfterClass
public static void tearDown() throws Exception {
	if (testCluster != null) {
		testCluster.stop();
	}

	TestStreamEnvironment.unsetAsContext();
	TestEnvironment.unsetAsContext();
}
 

import org.apache.flink.streaming.util.TestStreamEnvironment; //导入依赖的package包/类
@Test
public void testStreamingCustomSplitJobWithCustomClassLoader() throws IOException, ProgramInvocationException {
	PackagedProgram streamingInputSplitTestProg = new PackagedProgram(new File(STREAMING_INPUT_SPLITS_PROG_JAR_FILE));

	TestStreamEnvironment.setAsContext(
		testCluster,
		parallelism,
		Collections.singleton(new Path(STREAMING_INPUT_SPLITS_PROG_JAR_FILE)),
		Collections.<URL>emptyList());

	streamingInputSplitTestProg.invokeInteractiveModeForExecution();
}
 

import org.apache.flink.streaming.util.TestStreamEnvironment; //导入依赖的package包/类
@Test
public void testStreamingClassloaderJobWithCustomClassLoader() throws IOException, ProgramInvocationException {
	// regular streaming job
	PackagedProgram streamingProg = new PackagedProgram(new File(STREAMING_PROG_JAR_FILE));

	TestStreamEnvironment.setAsContext(
		testCluster,
		parallelism,
		Collections.singleton(new Path(STREAMING_PROG_JAR_FILE)),
		Collections.<URL>emptyList());

	streamingProg.invokeInteractiveModeForExecution();
}
 

import org.apache.flink.streaming.util.TestStreamEnvironment; //导入依赖的package包/类
@BeforeClass
public static void startTestCluster() {
	Configuration config = new Configuration();
	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
	config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, PARALLELISM / 2);
	config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 48L);
	config.setString(AkkaOptions.LOOKUP_TIMEOUT, "60 s");
	config.setString(AkkaOptions.ASK_TIMEOUT, "60 s");
	cluster = new LocalFlinkMiniCluster(config, false);
	cluster.start();

	env = new TestStreamEnvironment(cluster, PARALLELISM);
}
 

import org.apache.flink.streaming.util.TestStreamEnvironment; //导入依赖的package包/类
@BeforeClass
public static void startTestCluster() {
	Configuration config = new Configuration();
	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 2);
	config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, PARALLELISM / 2);
	config.setLong(TaskManagerOptions.MANAGED_MEMORY_SIZE, 48L);

	cluster = new LocalFlinkMiniCluster(config, false);
	cluster.start();

	env = new TestStreamEnvironment(cluster, PARALLELISM);
}
 

import org.apache.flink.streaming.util.TestStreamEnvironment; //导入依赖的package包/类
@Test
public void testCollect() throws Exception {
	final LocalFlinkMiniCluster cluster = new LocalFlinkMiniCluster(new Configuration(), false);
	try {
		cluster.start();

		TestStreamEnvironment.setAsContext(cluster, 1);

		final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

		final long n = 10;
		DataStream<Long> stream = env.generateSequence(1, n);

		long i = 1;
		for (Iterator<Long> it = DataStreamUtils.collect(stream); it.hasNext(); ) {
			long x = it.next();
			assertEquals("received wrong element", i, x);
			i++;
		}

		assertEquals("received wrong number of elements", n + 1, i);
	}
	finally {
		TestStreamEnvironment.unsetAsContext();
		cluster.stop();
	}
}