Java Configuration.getFloat方法代碼示例

import org.apache.flink.configuration.Configuration; //導入方法依賴的package包/類
/**
 * This tests that per default the off heap memory is set to what the network buffers require.
 */
@Test
public void testOffHeapMemoryWithDefaultConfiguration() {
	Configuration conf = new Configuration();

	ContaineredTaskManagerParameters params =
		ContaineredTaskManagerParameters.create(conf, CONTAINER_MEMORY, 1);

	final float memoryCutoffRatio = conf.getFloat(
		ConfigConstants.CONTAINERIZED_HEAP_CUTOFF_RATIO,
		ConfigConstants.DEFAULT_YARN_HEAP_CUTOFF_RATIO);
	final int minCutoff = conf.getInteger(
		ConfigConstants.CONTAINERIZED_HEAP_CUTOFF_MIN,
		ConfigConstants.DEFAULT_YARN_HEAP_CUTOFF);

	long cutoff = Math.max((long) (CONTAINER_MEMORY * memoryCutoffRatio), minCutoff);
	final long networkBufMB =
		calculateNetworkBufferMemory(
			(CONTAINER_MEMORY - cutoff) << 20, // megabytes to bytes
			conf) >> 20; // bytes to megabytes
	assertEquals(networkBufMB + cutoff, params.taskManagerDirectMemoryLimitMB());
}
 

import org.apache.flink.configuration.Configuration; //導入方法依賴的package包/類
/**
 * Calculates the amount of heap memory to use (to set via <tt>-Xmx</tt> and <tt>-Xms</tt>)
 * based on the total memory to use and the given configuration parameters.
 *
 * @param totalJavaMemorySizeMB
 * 		overall available memory to use (heap and off-heap)
 * @param config
 * 		configuration object
 *
 * @return heap memory to use (in megabytes)
 */
public static long calculateHeapSizeMB(long totalJavaMemorySizeMB, Configuration config) {
	Preconditions.checkArgument(totalJavaMemorySizeMB > 0);

	// subtract the Java memory used for network buffers (always off-heap)
	final long networkBufMB =
		calculateNetworkBufferMemory(
			totalJavaMemorySizeMB << 20, // megabytes to bytes
			config) >> 20; // bytes to megabytes
	final long remainingJavaMemorySizeMB = totalJavaMemorySizeMB - networkBufMB;

	// split the available Java memory between heap and off-heap

	final boolean useOffHeap = config.getBoolean(TaskManagerOptions.MEMORY_OFF_HEAP);

	final long heapSizeMB;
	if (useOffHeap) {

		long offHeapSize = config.getLong(TaskManagerOptions.MANAGED_MEMORY_SIZE);

		if (offHeapSize <= 0) {
			// calculate off-heap section via fraction
			double fraction = config.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);
			offHeapSize = (long) (fraction * remainingJavaMemorySizeMB);
		}

		TaskManagerServicesConfiguration
			.checkConfigParameter(offHeapSize < remainingJavaMemorySizeMB, offHeapSize,
				TaskManagerOptions.MANAGED_MEMORY_SIZE.key(),
				"Managed memory size too large for " + networkBufMB +
					" MB network buffer memory and a total of " + totalJavaMemorySizeMB +
					" MB JVM memory");

		heapSizeMB = remainingJavaMemorySizeMB - offHeapSize;
	} else {
		heapSizeMB = remainingJavaMemorySizeMB;
	}

	return heapSizeMB;
}
 

import org.apache.flink.configuration.Configuration; //導入方法依賴的package包/類
public JobGraphGenerator(Configuration config) {
	this.defaultMaxFan = config.getInteger(ConfigConstants.DEFAULT_SPILLING_MAX_FAN_KEY, 
		ConfigConstants.DEFAULT_SPILLING_MAX_FAN);
	this.defaultSortSpillingThreshold = config.getFloat(ConfigConstants.DEFAULT_SORT_SPILLING_THRESHOLD_KEY,
		ConfigConstants.DEFAULT_SORT_SPILLING_THRESHOLD);
	this.useLargeRecordHandler = config.getBoolean(
			ConfigConstants.USE_LARGE_RECORD_HANDLER_KEY,
			ConfigConstants.DEFAULT_USE_LARGE_RECORD_HANDLER);
}
 

import org.apache.flink.configuration.Configuration; //導入方法依賴的package包/類
/**
 * Calculates the amount of memory used for network buffers based on the total memory to use and
 * the according configuration parameters.
 *
 * <p>The following configuration parameters are involved:
 * <ul>
 *  <li>{@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_FRACTION},</li>
 * 	<li>{@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MIN},</li>
 * 	<li>{@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MAX}, and</li>
 *  <li>{@link TaskManagerOptions#NETWORK_NUM_BUFFERS} (fallback if the ones above do not exist)</li>
 * </ul>.
 *
 * @param totalJavaMemorySize
 * 		overall available memory to use (heap and off-heap, in bytes)
 * @param config
 * 		configuration object
 *
 * @return memory to use for network buffers (in bytes); at least one memory segment
 */
@SuppressWarnings("deprecation")
public static long calculateNetworkBufferMemory(long totalJavaMemorySize, Configuration config) {
	Preconditions.checkArgument(totalJavaMemorySize > 0);

	int segmentSize = config.getInteger(TaskManagerOptions.MEMORY_SEGMENT_SIZE);

	final long networkBufBytes;
	if (TaskManagerServicesConfiguration.hasNewNetworkBufConf(config)) {
		// new configuration based on fractions of available memory with selectable min and max
		float networkBufFraction = config.getFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION);
		long networkBufMin = config.getLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN);
		long networkBufMax = config.getLong(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX);

		TaskManagerServicesConfiguration
			.checkNetworkBufferConfig(segmentSize, networkBufFraction, networkBufMin, networkBufMax);

		networkBufBytes = Math.min(networkBufMax, Math.max(networkBufMin,
			(long) (networkBufFraction * totalJavaMemorySize)));

		TaskManagerServicesConfiguration
			.checkConfigParameter(networkBufBytes < totalJavaMemorySize,
				"(" + networkBufFraction + ", " + networkBufMin + ", " + networkBufMax + ")",
				"(" + TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key() + ", " +
					TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.key() + ", " +
					TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ")",
				"Network buffer memory size too large: " + networkBufBytes + " >= " +
					totalJavaMemorySize + " (total JVM memory size)");
		TaskManagerServicesConfiguration
			.checkConfigParameter(networkBufBytes >= segmentSize,
				"(" + networkBufFraction + ", " + networkBufMin + ", " + networkBufMax + ")",
				"(" + TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key() + ", " +
					TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.key() + ", " +
					TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ")",
				"Network buffer memory size too small: " + networkBufBytes + " < " +
					segmentSize + " (" + TaskManagerOptions.MEMORY_SEGMENT_SIZE.key() + ")");
	} else {
		// use old (deprecated) network buffers parameter
		int numNetworkBuffers = config.getInteger(TaskManagerOptions.NETWORK_NUM_BUFFERS);
		networkBufBytes = (long) numNetworkBuffers * (long) segmentSize;

		TaskManagerServicesConfiguration.checkNetworkConfigOld(numNetworkBuffers);

		TaskManagerServicesConfiguration
			.checkConfigParameter(networkBufBytes < totalJavaMemorySize,
				networkBufBytes, TaskManagerOptions.NETWORK_NUM_BUFFERS.key(),
				"Network buffer memory size too large: " + networkBufBytes + " >= " +
					totalJavaMemorySize + " (total JVM memory size)");
		TaskManagerServicesConfiguration
			.checkConfigParameter(networkBufBytes >= segmentSize,
				networkBufBytes, TaskManagerOptions.NETWORK_NUM_BUFFERS.key(),
				"Network buffer memory size too small: " + networkBufBytes + " < " +
					segmentSize + " (" + TaskManagerOptions.MEMORY_SEGMENT_SIZE.key() + ")");
	}

	return networkBufBytes;
}
 

import org.apache.flink.configuration.Configuration; //導入方法依賴的package包/類
/**
 * Utility method to extract TaskManager config parameters from the configuration and to
 * sanity check them.
 *
 * @param configuration The configuration.
 * @param remoteAddress identifying the IP address under which the TaskManager will be accessible
 * @param localCommunication True, to skip initializing the network stack.
 *                                      Use only in cases where only one task manager runs.
 * @return TaskExecutorConfiguration that wrappers InstanceConnectionInfo, NetworkEnvironmentConfiguration, etc.
 */
public static TaskManagerServicesConfiguration fromConfiguration(
		Configuration configuration,
		InetAddress remoteAddress,
		boolean localCommunication) throws Exception {

	// we need this because many configs have been written with a "-1" entry
	int slots = configuration.getInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 1);
	if (slots == -1) {
		slots = 1;
	}

	final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration);

	final NetworkEnvironmentConfiguration networkConfig = parseNetworkEnvironmentConfiguration(
		configuration,
		localCommunication,
		remoteAddress,
		slots);

	final QueryableStateConfiguration queryableStateConfig =
			parseQueryableStateConfiguration(configuration);

	// extract memory settings
	long configuredMemory = configuration.getLong(TaskManagerOptions.MANAGED_MEMORY_SIZE);
	checkConfigParameter(
		configuredMemory == TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue() ||
			configuredMemory > 0, configuredMemory,
		TaskManagerOptions.MANAGED_MEMORY_SIZE.key(),
		"MemoryManager needs at least one MB of memory. " +
			"If you leave this config parameter empty, the system automatically " +
			"pick a fraction of the available memory.");

	// check whether we use heap or off-heap memory
	final MemoryType memType;
	if (configuration.getBoolean(TaskManagerOptions.MEMORY_OFF_HEAP)) {
		memType = MemoryType.OFF_HEAP;
	} else {
		memType = MemoryType.HEAP;
	}

	boolean preAllocateMemory = configuration.getBoolean(TaskManagerOptions.MANAGED_MEMORY_PRE_ALLOCATE);

	float memoryFraction = configuration.getFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION);
	checkConfigParameter(memoryFraction > 0.0f && memoryFraction < 1.0f, memoryFraction,
		TaskManagerOptions.MANAGED_MEMORY_FRACTION.key(),
		"MemoryManager fraction of the free memory must be between 0.0 and 1.0");

	long timerServiceShutdownTimeout = AkkaUtils.getTimeout(configuration).toMillis();

	return new TaskManagerServicesConfiguration(
		remoteAddress,
		tmpDirs,
		networkConfig,
		queryableStateConfig,
		slots,
		configuredMemory,
		memType,
		preAllocateMemory,
		memoryFraction,
		timerServiceShutdownTimeout);
}