Java FlinkYarnSessionCli.getDynamicProperties方法代码示例

import org.apache.flink.yarn.cli.FlinkYarnSessionCli; //导入方法依赖的package包/类
@Test
public void testDynamicProperties() throws Exception {

	FlinkYarnSessionCli cli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"",
		"",
		false);
	Options options = new Options();
	cli.addGeneralOptions(options);
	cli.addRunOptions(options);

	CommandLineParser parser = new DefaultParser();
	CommandLine cmd = parser.parse(options, new String[]{"run", "-j", "fake.jar", "-n", "15",
			"-D", "akka.ask.timeout=5 min", "-D", "env.java.opts=-DappName=foobar"});

	AbstractYarnClusterDescriptor flinkYarnDescriptor = cli.createClusterDescriptor(cmd);

	Assert.assertNotNull(flinkYarnDescriptor);

	Map<String, String> dynProperties =
		FlinkYarnSessionCli.getDynamicProperties(flinkYarnDescriptor.getDynamicPropertiesEncoded());
	assertEquals(2, dynProperties.size());
	assertEquals("5 min", dynProperties.get("akka.ask.timeout"));
	assertEquals("-DappName=foobar", dynProperties.get("env.java.opts"));
}
 

import org.apache.flink.yarn.cli.FlinkYarnSessionCli; //导入方法依赖的package包/类
@Test
public void testDynamicProperties() throws Exception {

	FlinkYarnSessionCli cli = new FlinkYarnSessionCli(
		"",
		"",
		false);
	Options options = new Options();
	cli.addGeneralOptions(options);
	cli.addRunOptions(options);

	CommandLineParser parser = new DefaultParser();
	CommandLine cmd = parser.parse(options, new String[]{"run", "-j", "fake.jar", "-n", "15",
			"-D", "akka.ask.timeout=5 min", "-D", "env.java.opts=-DappName=foobar"});

	AbstractYarnClusterDescriptor flinkYarnDescriptor = cli.createDescriptor(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		null,
		cmd);

	Assert.assertNotNull(flinkYarnDescriptor);

	Map<String, String> dynProperties =
		FlinkYarnSessionCli.getDynamicProperties(flinkYarnDescriptor.getDynamicPropertiesEncoded());
	Assert.assertEquals(2, dynProperties.size());
	Assert.assertEquals("5 min", dynProperties.get("akka.ask.timeout"));
	Assert.assertEquals("-DappName=foobar", dynProperties.get("env.java.opts"));
}
 

import org.apache.flink.yarn.cli.FlinkYarnSessionCli; //导入方法依赖的package包/类
public static Configuration loadConfiguration(String workingDirectory, Map<String, String> env, Logger log) {
		Configuration configuration = GlobalConfiguration.loadConfiguration(workingDirectory);

		final String remoteKeytabPrincipal = env.get(YarnConfigKeys.KEYTAB_PRINCIPAL);

		final String zooKeeperNamespace = env.get(YarnConfigKeys.ENV_ZOOKEEPER_NAMESPACE);

		final Map<String, String> dynamicProperties = FlinkYarnSessionCli.getDynamicProperties(
			env.get(YarnConfigKeys.ENV_DYNAMIC_PROPERTIES));

		final String hostname = env.get(ApplicationConstants.Environment.NM_HOST.key());
		Preconditions.checkState(
			hostname != null,
			"ApplicationMaster hostname variable %s not set",
			ApplicationConstants.Environment.NM_HOST.key());

		configuration.setString(JobManagerOptions.ADDRESS, hostname);
		configuration.setString(RestOptions.REST_ADDRESS, hostname);

		// TODO: Support port ranges for the AM
//		final String portRange = configuration.getString(
//			ConfigConstants.YARN_APPLICATION_MASTER_PORT,
//			ConfigConstants.DEFAULT_YARN_JOB_MANAGER_PORT);

		for (Map.Entry<String, String> property : dynamicProperties.entrySet()) {
			configuration.setString(property.getKey(), property.getValue());
		}

		if (zooKeeperNamespace != null) {
			configuration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zooKeeperNamespace);
		}

		// if a web monitor shall be started, set the port to random binding
		if (configuration.getInteger(WebOptions.PORT, 0) >= 0) {
			configuration.setInteger(WebOptions.PORT, 0);
		}

		if (configuration.getInteger(RestOptions.REST_PORT) >= 0) {
			// set the REST port to 0 to select it randomly
			configuration.setInteger(RestOptions.REST_PORT, 0);
		}

		// if the user has set the deprecated YARN-specific config keys, we add the
		// corresponding generic config keys instead. that way, later code needs not
		// deal with deprecated config keys

		BootstrapTools.substituteDeprecatedConfigPrefix(configuration,
			ConfigConstants.YARN_APPLICATION_MASTER_ENV_PREFIX,
			ResourceManagerOptions.CONTAINERIZED_MASTER_ENV_PREFIX);

		BootstrapTools.substituteDeprecatedConfigPrefix(configuration,
			ConfigConstants.YARN_TASK_MANAGER_ENV_PREFIX,
			ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX);

		final String keytabPath;

		if (env.get(YarnConfigKeys.KEYTAB_PATH) == null) {
			keytabPath = null;
		} else {
			File f = new File(workingDirectory, Utils.KEYTAB_FILE_NAME);
			keytabPath = f.getAbsolutePath();
		}

		if (keytabPath != null && remoteKeytabPrincipal != null) {
			configuration.setString(SecurityOptions.KERBEROS_LOGIN_KEYTAB, keytabPath);
			configuration.setString(SecurityOptions.KERBEROS_LOGIN_PRINCIPAL, remoteKeytabPrincipal);
		}

		// configure local directory
		if (configuration.contains(CoreOptions.TMP_DIRS)) {
			log.info("Overriding YARN's temporary file directories with those " +
				"specified in the Flink config: " + configuration.getValue(CoreOptions.TMP_DIRS));
		}
		else {
			final String localDirs = env.get(ApplicationConstants.Environment.LOCAL_DIRS.key());
			log.info("Setting directories for temporary files to: {}", localDirs);
			configuration.setString(CoreOptions.TMP_DIRS, localDirs);
		}

		return configuration;
	}