Java ScheduledUnit类代码示例

import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; //导入依赖的package包/类
@Override
public CompletableFuture<LogicalSlot> allocateSlot(
		SlotRequestId slotRequestId,
		ScheduledUnit scheduledUnit,
		ResourceProfile resourceProfile,
		Collection<TaskManagerLocation> locationPreferences,
		boolean allowQueuedScheduling,
		Time timeout) {

	return internalAllocateSlot(
		slotRequestId,
		scheduledUnit,
		resourceProfile,
		locationPreferences,
		allowQueuedScheduling);
}
 

import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; //导入依赖的package包/类
@Override
public CompletableFuture<SimpleSlot> allocateSlot(
		ScheduledUnit task,
		boolean allowQueued,
		Collection<TaskManagerLocation> preferredLocations) {

	final SlotRequestID requestId = new SlotRequestID();
	CompletableFuture<SimpleSlot> slotFuture = gateway.allocateSlot(requestId, task, ResourceProfile.UNKNOWN, preferredLocations, timeout);
	slotFuture.whenComplete(
		(SimpleSlot slot, Throwable failure) -> {
			if (failure != null) {
				gateway.cancelSlotAllocation(requestId);
			}
	});
	return slotFuture;
}
 

import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; //导入依赖的package包/类
@Override
public CompletableFuture<LogicalSlot> allocateSlot(
		ScheduledUnit task,
		boolean allowQueued,
		Collection<TaskManagerLocation> preferredLocations) {
	final SlotContext slot;

	synchronized (slots) {
		if (slots.isEmpty()) {
			slot = null;
		} else {
			slot = slots.removeFirst();
		}
	}

	if (slot != null) {
		SimpleSlot result = new SimpleSlot(slot, this, 0);
		return CompletableFuture.completedFuture(result);
	}
	else {
		return FutureUtils.completedExceptionally(new NoResourceAvailableException());
	}
}
 

import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; //导入依赖的package包/类
@Override
public CompletableFuture<LogicalSlot> allocateSlot(
		ScheduledUnit task,
		boolean allowQueued,
		Collection<TaskManagerLocation> preferredLocations) {
	JobVertexID vertexId = task.getTaskToExecute().getVertex().getJobvertexId();
	int subtask = task.getTaskToExecute().getParallelSubtaskIndex();

	CompletableFuture<LogicalSlot>[] forTask = slotFutures.get(vertexId);
	if (forTask != null) {
		CompletableFuture<LogicalSlot> future = forTask[subtask];
		if (future != null) {
			slotFutureRequested.get(vertexId)[subtask].complete(true);

			return future;
		}
	}

	throw new IllegalArgumentException("No registered slot future for task " + vertexId + " (" + subtask + ')');
}
 

import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; //导入依赖的package包/类
@Test
public void testScheduleToDeploy() {
	try {
		// a slot than cannot be deployed to
		final TaskOperationProtocol taskManager = mock(TaskOperationProtocol.class);
		final Instance instance = getInstance(taskManager);
		final AllocatedSlot slot = instance.allocateSlot(new JobID());
		
		final ExecutionJobVertex ejv = getJobVertexNotExecuting(new JobVertexID());
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0]);
		
		Scheduler scheduler = mock(Scheduler.class);
		when(scheduler.scheduleImmediately(Matchers.any(ScheduledUnit.class))).thenReturn(slot);
		
		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());

		// try to deploy to the slot
		vertex.scheduleForExecution(scheduler, false);
		assertEquals(ExecutionState.DEPLOYING, vertex.getExecutionState());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; //导入依赖的package包/类
CompletableFuture<SimpleSlot> internalAllocateSlot(
		SlotRequestID requestId,
		ScheduledUnit task,
		ResourceProfile resources,
		Iterable<TaskManagerLocation> locationPreferences) {

	// (1) do we have a slot available already?
	SlotAndLocality slotFromPool = availableSlots.poll(resources, locationPreferences);
	if (slotFromPool != null) {
		SimpleSlot slot = createSimpleSlot(slotFromPool.slot(), slotFromPool.locality());
		allocatedSlots.add(requestId, slot);
		return CompletableFuture.completedFuture(slot);
	}

	// the request will be completed by a future
	final CompletableFuture<SimpleSlot> future = new CompletableFuture<>();

	// (2) need to request a slot
	if (resourceManagerGateway == null) {
		// no slot available, and no resource manager connection
		stashRequestWaitingForResourceManager(requestId, resources, future);
	} else {
		// we have a resource manager connection, so let's ask it for more resources
		requestSlotFromResourceManager(new PendingRequest(requestId, future, resources));
	}

	return future;
}
 

import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; //导入依赖的package包/类
@Test
public void testSingleQueuedSharedSlotScheduling() throws Exception {
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
	slotPoolGateway.registerTaskManager(taskManagerLocation.getResourceID()).get();

	SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId();
	CompletableFuture<LogicalSlot> logicalSlotFuture = slotProvider.allocateSlot(
		new ScheduledUnit(
			new JobVertexID(),
			slotSharingGroupId,
			null),
		true,
		Collections.emptyList());

	assertFalse(logicalSlotFuture.isDone());

	final AllocationID allocationId = allocationIdFuture.get();

	CompletableFuture<Boolean> booleanCompletableFuture = slotPoolGateway.offerSlot(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		new SlotOffer(
			allocationId,
			0,
			ResourceProfile.UNKNOWN));

	assertTrue(booleanCompletableFuture.get());

	final LogicalSlot logicalSlot = logicalSlotFuture.get();

	assertEquals(slotSharingGroupId, logicalSlot.getSlotSharingGroupId());
}
 

import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; //导入依赖的package包/类
/**
 * Tests that returned slot futures are failed if the allocation request is failed.
 */
@Test
public void testFailingQueuedSharedSlotScheduling() throws ExecutionException, InterruptedException {
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	CompletableFuture<LogicalSlot> logicalSlotFuture = slotProvider.allocateSlot(
		new ScheduledUnit(
			new JobVertexID(),
			new SlotSharingGroupId(),
			null),
		true,
		Collections.emptyList());

	final AllocationID allocationId = allocationIdFuture.get();

	// this should fail the returned logical slot future
	slotPoolGateway.failAllocation(allocationId, new FlinkException("Testing Exception"));

	try {
		logicalSlotFuture.get();
		fail("The slot future should have failed.");
	} catch (ExecutionException ee) {
		assertTrue(ExceptionUtils.findThrowable(ee, FlinkException.class).isPresent());
	}
}
 

import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; //导入依赖的package包/类
@Test
public void testSlotAllocationNoResourceManager() throws Exception {
	final JobID jid = new JobID();
	
	final SlotPool pool = new SlotPool(
		rpcService,
		jid,
		SystemClock.getInstance(),
		TestingUtils.infiniteTime(),
		TestingUtils.infiniteTime(),
		Time.milliseconds(10L) // this is the timeout for the request tested here
	);

	try {
		pool.start(JobMasterId.generate(), "foobar");

		CompletableFuture<LogicalSlot> future = pool.allocateSlot(
			new SlotRequestId(),
			new ScheduledUnit(SchedulerTestUtils.getDummyTask()),
			DEFAULT_TESTING_PROFILE,
			Collections.emptyList(),
			true,
			TestingUtils.infiniteTime());

		try {
			future.get();
			fail("We expected an ExecutionException.");
		} catch (ExecutionException e) {
			assertTrue(ExceptionUtils.stripExecutionException(e) instanceof NoResourceAvailableException);
		}
	} finally {
		RpcUtils.terminateRpcEndpoint(pool, timeout);
	}
}
 

import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; //导入依赖的package包/类
@Test
public void testAllocateSimpleSlot() throws Exception {
	ResourceManagerGateway resourceManagerGateway = createResourceManagerGatewayMock();
	final SlotPool slotPool = new SlotPool(rpcService, jobId);

	try {
		SlotPoolGateway slotPoolGateway = setupSlotPool(slotPool, resourceManagerGateway);
		ResourceID resourceID = new ResourceID("resource");
		slotPoolGateway.registerTaskManager(resourceID);

		SlotPoolGateway.SlotRequestID requestId = new SlotPoolGateway.SlotRequestID();
		CompletableFuture<SimpleSlot> future = slotPoolGateway.allocateSlot(requestId, mock(ScheduledUnit.class), DEFAULT_TESTING_PROFILE, null, timeout);
		assertFalse(future.isDone());

		ArgumentCaptor<SlotRequest> slotRequestArgumentCaptor = ArgumentCaptor.forClass(SlotRequest.class);
		verify(resourceManagerGateway, Mockito.timeout(timeout.toMilliseconds())).requestSlot(any(JobMasterId.class), slotRequestArgumentCaptor.capture(), any(Time.class));

		final SlotRequest slotRequest = slotRequestArgumentCaptor.getValue();

		AllocatedSlot allocatedSlot = createAllocatedSlot(resourceID, slotRequest.getAllocationId(), jobId, DEFAULT_TESTING_PROFILE);
		assertTrue(slotPoolGateway.offerSlot(allocatedSlot).get());

		SimpleSlot slot = future.get(1, TimeUnit.SECONDS);
		assertTrue(future.isDone());
		assertTrue(slot.isAlive());
		assertEquals(resourceID, slot.getTaskManagerID());
		assertEquals(jobId, slot.getJobID());
		assertEquals(slotPool.getSlotOwner(), slot.getOwner());
		assertEquals(slotPool.getAllocatedSlots().get(slot.getAllocatedSlot().getSlotAllocationId()), slot);
	} finally {
		slotPool.shutDown();
	}
}
 

import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; //导入依赖的package包/类
@Test
public void testSlotAllocationNoResourceManager() throws Exception {
	final JobID jid = new JobID();
	
	final SlotPool pool = new SlotPool(
		rpcService,
		jid,
		SystemClock.getInstance(),
		TestingUtils.infiniteTime(),
		TestingUtils.infiniteTime(),
		Time.milliseconds(10L) // this is the timeout for the request tested here
	);

	try {
		pool.start(JobMasterId.generate(), "foobar");

		CompletableFuture<SimpleSlot> future = pool.allocateSlot(
			new SlotPoolGateway.SlotRequestID(),
			new ScheduledUnit(SchedulerTestUtils.getDummyTask()),
			DEFAULT_TESTING_PROFILE,
			Collections.emptyList(),
			TestingUtils.infiniteTime());

		try {
			future.get();
			fail("We expected an ExecutionException.");
		} catch (ExecutionException e) {
			assertTrue(ExceptionUtils.stripExecutionException(e) instanceof NoResourceAvailableException);
		}
	} finally {
		RpcUtils.terminateRpcEndpoint(pool, timeout);
	}
}
 

import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; //导入依赖的package包/类
@Test
public void testCancelSlotAllocationWithoutResourceManager() throws Exception {
	final JobID jid = new JobID();

	final TestingSlotPool pool = new TestingSlotPool(
		rpcService,
		jid,
		SystemClock.getInstance(),
		TestingUtils.infiniteTime(),
		TestingUtils.infiniteTime(),
		TestingUtils.infiniteTime());

	try {
		pool.start(JobMasterId.generate(), "foobar");
		SlotPoolGateway slotPoolGateway = pool.getSelfGateway(SlotPoolGateway.class);

		SlotPoolGateway.SlotRequestID requestId = new SlotPoolGateway.SlotRequestID();
		CompletableFuture<SimpleSlot> future = slotPoolGateway.allocateSlot(
			requestId,
			new ScheduledUnit(SchedulerTestUtils.getDummyTask()),
			DEFAULT_TESTING_PROFILE,
			Collections.emptyList(),
			Time.milliseconds(10L));

		try {
			future.get();
			fail("We expected a AskTimeoutException.");
		} catch (ExecutionException e) {
			assertTrue(ExceptionUtils.stripExecutionException(e) instanceof AskTimeoutException);
		}

		assertEquals(1L, (long) pool.getNumberOfWaitingForResourceRequests().get());

		slotPoolGateway.cancelSlotAllocation(requestId).get();

		assertEquals(0L, (long) pool.getNumberOfWaitingForResourceRequests().get());
	} finally {
		RpcUtils.terminateRpcEndpoint(pool, timeout);
	}
}
 

import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; //导入依赖的package包/类
@Test
public void testSlotReleasedWhenScheduledImmediately() {
	try {
		final ExecutionJobVertex ejv = getExecutionVertex(new JobVertexID());
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
				AkkaUtils.getDefaultTimeout());

		// a slot than cannot be deployed to
		final Instance instance = getInstance(new ActorTaskManagerGateway(DummyActorGateway.INSTANCE));
		final SimpleSlot slot = instance.allocateSimpleSlot();
		
		slot.releaseSlot();
		assertTrue(slot.isReleased());

		Scheduler scheduler = mock(Scheduler.class);
		CompletableFuture<SimpleSlot> future = new CompletableFuture<>();
		future.complete(slot);
		when(scheduler.allocateSlot(Matchers.any(ScheduledUnit.class), anyBoolean(), any(Collection.class))).thenReturn(future);

		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
		// try to deploy to the slot
		vertex.scheduleForExecution(scheduler, false, LocationPreferenceConstraint.ALL);

		// will have failed
		assertEquals(ExecutionState.FAILED, vertex.getExecutionState());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; //导入依赖的package包/类
@Test
public void testSlotReleasedWhenScheduledQueued() {
	try {
		final ExecutionJobVertex ejv = getExecutionVertex(new JobVertexID());
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
				AkkaUtils.getDefaultTimeout());

		// a slot than cannot be deployed to
		final Instance instance = getInstance(new ActorTaskManagerGateway(DummyActorGateway.INSTANCE));
		final SimpleSlot slot = instance.allocateSimpleSlot();

		slot.releaseSlot();
		assertTrue(slot.isReleased());

		final CompletableFuture<SimpleSlot> future = new CompletableFuture<>();

		Scheduler scheduler = mock(Scheduler.class);
		when(scheduler.allocateSlot(Matchers.any(ScheduledUnit.class), anyBoolean(), any(Collection.class))).thenReturn(future);

		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
		// try to deploy to the slot
		vertex.scheduleForExecution(scheduler, true, LocationPreferenceConstraint.ALL);

		// future has not yet a slot
		assertEquals(ExecutionState.SCHEDULED, vertex.getExecutionState());

		future.complete(slot);

		// will have failed
		assertEquals(ExecutionState.FAILED, vertex.getExecutionState());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

import org.apache.flink.runtime.jobmanager.scheduler.ScheduledUnit; //导入依赖的package包/类
@Test
public void testScheduleToDeploying() {
	try {
		final ExecutionJobVertex ejv = getExecutionVertex(new JobVertexID());
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
				AkkaUtils.getDefaultTimeout());

		final Instance instance = getInstance(new ActorTaskManagerGateway(
			new ExecutionGraphTestUtils.SimpleActorGateway(TestingUtils.defaultExecutionContext())));
		final SimpleSlot slot = instance.allocateSimpleSlot();

		Scheduler scheduler = mock(Scheduler.class);
		CompletableFuture<SimpleSlot> future = new CompletableFuture<>();
		future.complete(slot);
		when(scheduler.allocateSlot(Matchers.any(ScheduledUnit.class), anyBoolean(), any(Collection.class))).thenReturn(future);

		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());

		// try to deploy to the slot
		vertex.scheduleForExecution(scheduler, false, LocationPreferenceConstraint.ALL);
		assertEquals(ExecutionState.DEPLOYING, vertex.getExecutionState());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}