Java Scheduler类代码示例

import org.apache.flink.runtime.jobmanager.scheduler.Scheduler; //导入依赖的package包/类
private static Tuple2<ExecutionGraph, Instance> createExecutionGraph(RestartStrategy restartStrategy, boolean isSpy) throws Exception {
	Instance instance = ExecutionGraphTestUtils.getInstance(
		new ActorTaskManagerGateway(
			new SimpleActorGateway(TestingUtils.directExecutionContext())),
		NUM_TASKS);

	Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
	scheduler.newInstanceAvailable(instance);

	JobVertex sender = newJobVertex("Task", NUM_TASKS, NoOpInvokable.class);

	JobGraph jobGraph = new JobGraph("Pointwise job", sender);

	ExecutionGraph eg = newExecutionGraph(restartStrategy, scheduler);
	if (isSpy) {
		eg = spy(eg);
	}
	eg.attachJobGraph(jobGraph.getVerticesSortedTopologicallyFromSources());

	assertEquals(JobStatus.CREATED, eg.getState());

	eg.scheduleForExecution();
	assertEquals(JobStatus.RUNNING, eg.getState());
	return new Tuple2<>(eg, instance);
}
 

import org.apache.flink.runtime.jobmanager.scheduler.Scheduler; //导入依赖的package包/类
public static ExecutionJobVertex getExecutionVertex(
		JobVertexID id, ScheduledExecutorService executor) 
	throws Exception {

	JobVertex ajv = new JobVertex("TestVertex", id);
	ajv.setInvokableClass(mock(AbstractInvokable.class).getClass());

	ExecutionGraph graph = new ExecutionGraph(
		executor,
		executor,
		new JobID(), 
		"test job", 
		new Configuration(),
		new SerializedValue<>(new ExecutionConfig()),
		AkkaUtils.getDefaultTimeout(),
		new NoRestartStrategy(),
		new Scheduler(ExecutionContext$.MODULE$.fromExecutor(executor)));

	return spy(new ExecutionJobVertex(graph, ajv, 1, AkkaUtils.getDefaultTimeout()));
}
 

import org.apache.flink.runtime.jobmanager.scheduler.Scheduler; //导入依赖的package包/类
public void scheduleForExecution(Scheduler scheduler) throws JobException {
	if (scheduler == null) {
		throw new IllegalArgumentException("Scheduler must not be null.");
	}
	
	if (this.scheduler != null && this.scheduler != scheduler) {
		throw new IllegalArgumentException("Cann not use different schedulers for the same job");
	}
	
	if (transitionState(JobStatus.CREATED, JobStatus.RUNNING)) {
		this.scheduler = scheduler;
		
		// initially, we simply take the ones without inputs.
		// next, we implement the logic to go back from vertices that need computation
		// to the ones we need to start running
		for (ExecutionJobVertex ejv : this.tasks.values()) {
			if (ejv.getJobVertex().isInputVertex()) {
				ejv.scheduleAll(scheduler, allowQueuedScheduling);
			}
		}
	}
	else {
		throw new IllegalStateException("Job may only be scheduled from state " + JobStatus.CREATED);
	}
}
 

import org.apache.flink.runtime.jobmanager.scheduler.Scheduler; //导入依赖的package包/类
@Test
public void testScheduleToDeploy() {
	try {
		// a slot than cannot be deployed to
		final TaskOperationProtocol taskManager = mock(TaskOperationProtocol.class);
		final Instance instance = getInstance(taskManager);
		final AllocatedSlot slot = instance.allocateSlot(new JobID());
		
		final ExecutionJobVertex ejv = getJobVertexNotExecuting(new JobVertexID());
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0]);
		
		Scheduler scheduler = mock(Scheduler.class);
		when(scheduler.scheduleImmediately(Matchers.any(ScheduledUnit.class))).thenReturn(slot);
		
		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());

		// try to deploy to the slot
		vertex.scheduleForExecution(scheduler, false);
		assertEquals(ExecutionState.DEPLOYING, vertex.getExecutionState());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

import org.apache.flink.runtime.jobmanager.scheduler.Scheduler; //导入依赖的package包/类
private ExecutionGraph createExecutionGraphAndEnableCheckpointing(
		CheckpointIDCounter counter,
		CompletedCheckpointStore store) throws Exception {
	ExecutionGraph executionGraph = new ExecutionGraph(
		new DummyJobInformation(),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		Time.days(1L),
		new NoRestartStrategy(),
		new RestartAllStrategy.Factory(),
		new Scheduler(TestingUtils.defaultExecutionContext()),
		ClassLoader.getSystemClassLoader(),
		VoidBlobWriter.getInstance());

	executionGraph.enableCheckpointing(
			100,
			100,
			100,
			1,
			CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
			Collections.emptyList(),
			Collections.emptyList(),
			Collections.emptyList(),
			Collections.emptyList(),
			counter,
			store,
			new MemoryStateBackend(),
			CheckpointStatsTrackerTest.createTestTracker());

	JobVertex jobVertex = new JobVertex("MockVertex");
	jobVertex.setInvokableClass(AbstractInvokable.class);
	executionGraph.attachJobGraph(Collections.singletonList(jobVertex));

	return executionGraph;
}
 

import org.apache.flink.runtime.jobmanager.scheduler.Scheduler; //导入依赖的package包/类
private Props createJobManagerProps(Configuration configuration) throws Exception {
	LeaderElectionService leaderElectionService;
	if (HighAvailabilityMode.fromConfig(configuration) == HighAvailabilityMode.NONE) {
		leaderElectionService = new StandaloneLeaderElectionService();
	}
	else {
		CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
		leaderElectionService = ZooKeeperUtils.createLeaderElectionService(client,
				configuration);
	}

	// We don't need recovery in this test
	SubmittedJobGraphStore submittedJobGraphStore = new StandaloneSubmittedJobGraphStore();
	CheckpointRecoveryFactory checkpointRecoveryFactory = new StandaloneCheckpointRecoveryFactory();

	configuration.setLong(BlobServerOptions.CLEANUP_INTERVAL, 1L);

	BlobServer blobServer = new BlobServer(configuration, new VoidBlobStore());
	blobServer.start();
	return Props.create(
		TestingJobManager.class,
		configuration,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		new InstanceManager(),
		new Scheduler(TestingUtils.defaultExecutionContext()),
		blobServer,
		new BlobLibraryCacheManager(blobServer, FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST, new String[0]),
		ActorRef.noSender(),
		new NoRestartStrategy.NoRestartStrategyFactory(),
		AkkaUtils.getDefaultTimeoutAsFiniteDuration(),
		leaderElectionService,
		submittedJobGraphStore,
		checkpointRecoveryFactory,
		AkkaUtils.getDefaultTimeoutAsFiniteDuration(),
		UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(),
		Option.<String>empty());
}
 

import org.apache.flink.runtime.jobmanager.scheduler.Scheduler; //导入依赖的package包/类
/**
 * Tests that it is possible to fail a graph via a call to
 * {@link ExecutionGraph#failGlobal(Throwable)} after cancellation.
 */
@Test
public void testFailExecutionGraphAfterCancel() throws Exception {
	Instance instance = ExecutionGraphTestUtils.getInstance(
		new ActorTaskManagerGateway(
			new SimpleActorGateway(TestingUtils.directExecutionContext())),
		2);

	Scheduler scheduler = new Scheduler(TestingUtils.defaultExecutionContext());
	scheduler.newInstanceAvailable(instance);

	JobVertex vertex = newJobVertex("Test Vertex", 1, NoOpInvokable.class);

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(
		Integer.MAX_VALUE, Integer.MAX_VALUE));
	JobGraph jobGraph = new JobGraph("Test Job", vertex);
	jobGraph.setExecutionConfig(executionConfig);

	ExecutionGraph eg = newExecutionGraph(new InfiniteDelayRestartStrategy(), scheduler);

	eg.attachJobGraph(jobGraph.getVerticesSortedTopologicallyFromSources());

	assertEquals(JobStatus.CREATED, eg.getState());

	eg.scheduleForExecution();
	assertEquals(JobStatus.RUNNING, eg.getState());

	// Fail right after cancel (for example with concurrent slot release)
	eg.cancel();
	assertEquals(JobStatus.CANCELLING, eg.getState());

	eg.failGlobal(new Exception("Test Exception"));
	assertEquals(JobStatus.FAILING, eg.getState());

	Execution execution = eg.getAllExecutionVertices().iterator().next().getCurrentExecutionAttempt();

	execution.cancelingComplete();
	assertEquals(JobStatus.RESTARTING, eg.getState());
}
 

import org.apache.flink.runtime.jobmanager.scheduler.Scheduler; //导入依赖的package包/类
private Scheduler createSchedulerWithInstances(int num, TaskManagerGateway taskManagerGateway) {
	final Scheduler scheduler = new Scheduler(executor);
	final Instance[] instances = new Instance[num];

	for (int i = 0; i < instances.length; i++) {
		instances[i] = createInstance(taskManagerGateway, 55443 + i);
		scheduler.newInstanceAvailable(instances[i]);
	}

	return scheduler;
}
 

import org.apache.flink.runtime.jobmanager.scheduler.Scheduler; //导入依赖的package包/类
private static ExecutionGraph newExecutionGraph(RestartStrategy restartStrategy, Scheduler scheduler) throws IOException {
	return new ExecutionGraph(
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		new JobID(),
		"Test job",
		new Configuration(),
		new SerializedValue<>(new ExecutionConfig()),
		AkkaUtils.getDefaultTimeout(),
		restartStrategy,
		scheduler);
}
 

import org.apache.flink.runtime.jobmanager.scheduler.Scheduler; //导入依赖的package包/类
TestExecGraph(JobID jobId) throws IOException {
	super(
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		jobId,
		"test graph",
		EMPTY_CONFIG,
		new SerializedValue<>(new ExecutionConfig()),
		TIMEOUT,
		new FixedDelayRestartStrategy(1, 0),
		new Scheduler(TestingUtils.defaultExecutionContext()));
}
 

import org.apache.flink.runtime.jobmanager.scheduler.Scheduler; //导入依赖的package包/类
@Test
public void testSlotReleasedWhenScheduledImmediately() {
	try {
		final ExecutionJobVertex ejv = getExecutionVertex(new JobVertexID());
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
				AkkaUtils.getDefaultTimeout());

		// a slot than cannot be deployed to
		final Instance instance = getInstance(new ActorTaskManagerGateway(DummyActorGateway.INSTANCE));
		final SimpleSlot slot = instance.allocateSimpleSlot();
		
		slot.releaseSlot();
		assertTrue(slot.isReleased());

		Scheduler scheduler = mock(Scheduler.class);
		CompletableFuture<SimpleSlot> future = new CompletableFuture<>();
		future.complete(slot);
		when(scheduler.allocateSlot(Matchers.any(ScheduledUnit.class), anyBoolean(), any(Collection.class))).thenReturn(future);

		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
		// try to deploy to the slot
		vertex.scheduleForExecution(scheduler, false, LocationPreferenceConstraint.ALL);

		// will have failed
		assertEquals(ExecutionState.FAILED, vertex.getExecutionState());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

import org.apache.flink.runtime.jobmanager.scheduler.Scheduler; //导入依赖的package包/类
@Test
public void testSlotReleasedWhenScheduledQueued() {
	try {
		final ExecutionJobVertex ejv = getExecutionVertex(new JobVertexID());
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
				AkkaUtils.getDefaultTimeout());

		// a slot than cannot be deployed to
		final Instance instance = getInstance(new ActorTaskManagerGateway(DummyActorGateway.INSTANCE));
		final SimpleSlot slot = instance.allocateSimpleSlot();

		slot.releaseSlot();
		assertTrue(slot.isReleased());

		final CompletableFuture<SimpleSlot> future = new CompletableFuture<>();

		Scheduler scheduler = mock(Scheduler.class);
		when(scheduler.allocateSlot(Matchers.any(ScheduledUnit.class), anyBoolean(), any(Collection.class))).thenReturn(future);

		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
		// try to deploy to the slot
		vertex.scheduleForExecution(scheduler, true, LocationPreferenceConstraint.ALL);

		// future has not yet a slot
		assertEquals(ExecutionState.SCHEDULED, vertex.getExecutionState());

		future.complete(slot);

		// will have failed
		assertEquals(ExecutionState.FAILED, vertex.getExecutionState());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

import org.apache.flink.runtime.jobmanager.scheduler.Scheduler; //导入依赖的package包/类
@Test
public void testScheduleToDeploying() {
	try {
		final ExecutionJobVertex ejv = getExecutionVertex(new JobVertexID());
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
				AkkaUtils.getDefaultTimeout());

		final Instance instance = getInstance(new ActorTaskManagerGateway(
			new ExecutionGraphTestUtils.SimpleActorGateway(TestingUtils.defaultExecutionContext())));
		final SimpleSlot slot = instance.allocateSimpleSlot();

		Scheduler scheduler = mock(Scheduler.class);
		CompletableFuture<SimpleSlot> future = new CompletableFuture<>();
		future.complete(slot);
		when(scheduler.allocateSlot(Matchers.any(ScheduledUnit.class), anyBoolean(), any(Collection.class))).thenReturn(future);

		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());

		// try to deploy to the slot
		vertex.scheduleForExecution(scheduler, false, LocationPreferenceConstraint.ALL);
		assertEquals(ExecutionState.DEPLOYING, vertex.getExecutionState());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

import org.apache.flink.runtime.jobmanager.scheduler.Scheduler; //导入依赖的package包/类
public TestingFailingHAJobManager(
	Configuration flinkConfiguration,
	ScheduledExecutorService futureExecutor,
	Executor ioExecutor,
	InstanceManager instanceManager,
	Scheduler scheduler,
	BlobServer blobServer,
	BlobLibraryCacheManager libraryCacheManager,
	ActorRef archive,
	RestartStrategyFactory restartStrategyFactory,
	FiniteDuration timeout,
	LeaderElectionService leaderElectionService,
	SubmittedJobGraphStore submittedJobGraphs,
	CheckpointRecoveryFactory checkpointRecoveryFactory,
	FiniteDuration jobRecoveryTimeout,
	JobManagerMetricGroup jobManagerMetricGroup,
	Collection<JobID> recoveredJobs) {
	super(
		flinkConfiguration,
		futureExecutor,
		ioExecutor,
		instanceManager,
		scheduler,
		blobServer,
		libraryCacheManager,
		archive,
		restartStrategyFactory,
		timeout,
		leaderElectionService,
		submittedJobGraphs,
		checkpointRecoveryFactory,
		jobRecoveryTimeout,
		jobManagerMetricGroup,
		Option.empty());

	this.recoveredJobs = recoveredJobs;
}
 

import org.apache.flink.runtime.jobmanager.scheduler.Scheduler; //导入依赖的package包/类
@Test
public void testSlotReleasedWhenScheduledImmediately() {
	
	try {
		// a slot than cannot be deployed to
		final TaskOperationProtocol taskManager = mock(TaskOperationProtocol.class);
		final Instance instance = getInstance(taskManager);
		final AllocatedSlot slot = instance.allocateSlot(new JobID());
		slot.cancel();
		assertFalse(slot.isReleased());
		
		final ExecutionJobVertex ejv = getJobVertexNotExecuting(new JobVertexID());
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0]);
		
		Scheduler scheduler = mock(Scheduler.class);
		when(scheduler.scheduleImmediately(Matchers.any(ScheduledUnit.class))).thenReturn(slot);
		
		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
		// try to deploy to the slot
		vertex.scheduleForExecution(scheduler, false);
		
		// will have failed
		assertEquals(ExecutionState.FAILED, vertex.getExecutionState());
		assertTrue(slot.isReleased());
		
		verify(taskManager, times(0)).submitTask(Matchers.any(TaskDeploymentDescriptor.class));
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}