本文整理汇总了Python中airflow.contrib.hooks.spark_submit_hook.SparkSubmitHook._build_spark_submit_command方法的典型用法代码示例。如果您正苦于以下问题：Python SparkSubmitHook._build_spark_submit_command方法的具体用法？Python SparkSubmitHook._build_spark_submit_command怎么用？Python SparkSubmitHook._build_spark_submit_command使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类airflow.contrib.hooks.spark_submit_hook.SparkSubmitHook的用法示例。
在下文中一共展示了SparkSubmitHook._build_spark_submit_command方法的11个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: env_vars_exception_in_standalone_cluster_mode
# 需要导入模块: from airflow.contrib.hooks.spark_submit_hook import SparkSubmitHook [as 别名]
# 或者: from airflow.contrib.hooks.spark_submit_hook.SparkSubmitHook import _build_spark_submit_command [as 别名]
def env_vars_exception_in_standalone_cluster_mode():
# Given
hook = SparkSubmitHook(conn_id='spark_standalone_cluster',
env_vars={"bar": "foo"})
# When
hook._build_spark_submit_command(self._spark_job_file)示例2: test_resolve_spark_submit_env_vars_standalone_client_mode
# 需要导入模块: from airflow.contrib.hooks.spark_submit_hook import SparkSubmitHook [as 别名]
# 或者: from airflow.contrib.hooks.spark_submit_hook.SparkSubmitHook import _build_spark_submit_command [as 别名]
def test_resolve_spark_submit_env_vars_standalone_client_mode(self):
# Given
hook = SparkSubmitHook(conn_id='spark_standalone_cluster_client_mode',
env_vars={"bar": "foo"})
# When
hook._build_spark_submit_command(self._spark_job_file)
# Then
self.assertEqual(hook._env, {"bar": "foo"})示例3: test_build_spark_submit_command
# 需要导入模块: from airflow.contrib.hooks.spark_submit_hook import SparkSubmitHook [as 别名]
# 或者: from airflow.contrib.hooks.spark_submit_hook.SparkSubmitHook import _build_spark_submit_command [as 别名]
def test_build_spark_submit_command(self):
# Given
hook = SparkSubmitHook(**self._config)
# When
cmd = hook._build_spark_submit_command(self._spark_job_file)
# Then
expected_build_cmd = [
'spark-submit',
'--master', 'yarn',
'--conf', 'parquet.compression=SNAPPY',
'--files', 'hive-site.xml',
'--py-files', 'sample_library.py',
'--jars', 'parquet.jar',
'--packages', 'com.databricks:spark-avro_2.11:3.2.0',
'--exclude-packages', 'org.bad.dependency:1.0.0',
'--repositories', 'http://myrepo.org',
'--num-executors', '10',
'--total-executor-cores', '4',
'--executor-cores', '4',
'--executor-memory', '22g',
'--driver-memory', '3g',
'--keytab', 'privileged_user.keytab',
'--principal', 'user/[email protected]',
'--name', 'spark-job',
'--class', 'com.foo.bar.AppMain',
'--verbose',
'test_application.py',
'-f', 'foo',
'--bar', 'bar',
'--with-spaces', 'args should keep embdedded spaces',
'baz'
]
self.assertEquals(expected_build_cmd, cmd)示例4: test_resolve_spark_submit_env_vars_k8s
# 需要导入模块: from airflow.contrib.hooks.spark_submit_hook import SparkSubmitHook [as 别名]
# 或者: from airflow.contrib.hooks.spark_submit_hook.SparkSubmitHook import _build_spark_submit_command [as 别名]
def test_resolve_spark_submit_env_vars_k8s(self):
# Given
hook = SparkSubmitHook(conn_id='spark_k8s_cluster',
env_vars={"bar": "foo"})
# When
cmd = hook._build_spark_submit_command(self._spark_job_file)
# Then
self.assertEqual(cmd[4], "spark.kubernetes.driverEnv.bar=foo")示例5: test_resolve_connection_spark_standalone_cluster_connection
# 需要导入模块: from airflow.contrib.hooks.spark_submit_hook import SparkSubmitHook [as 别名]
# 或者: from airflow.contrib.hooks.spark_submit_hook.SparkSubmitHook import _build_spark_submit_command [as 别名]
def test_resolve_connection_spark_standalone_cluster_connection(self):
# Given
hook = SparkSubmitHook(conn_id='spark_standalone_cluster')
# When
connection = hook._resolve_connection()
cmd = hook._build_spark_submit_command(self._spark_job_file)
# Then
expected_spark_connection = {"master": "spark://spark-standalone-master:6066",
"spark_binary": "spark-submit",
"deploy_mode": "cluster",
"queue": None,
"spark_home": "/path/to/spark_home"}
self.assertEqual(connection, expected_spark_connection)
self.assertEqual(cmd[0], '/path/to/spark_home/bin/spark-submit')示例6: test_resolve_connection_spark_binary_set_connection
# 需要导入模块: from airflow.contrib.hooks.spark_submit_hook import SparkSubmitHook [as 别名]
# 或者: from airflow.contrib.hooks.spark_submit_hook.SparkSubmitHook import _build_spark_submit_command [as 别名]
def test_resolve_connection_spark_binary_set_connection(self):
# Given
hook = SparkSubmitHook(conn_id='spark_binary_set')
# When
connection = hook._resolve_connection()
cmd = hook._build_spark_submit_command(self._spark_job_file)
# Then
expected_spark_connection = {"master": "yarn",
"spark_binary": "custom-spark-submit",
"deploy_mode": None,
"queue": None,
"spark_home": None}
self.assertEqual(connection, expected_spark_connection)
self.assertEqual(cmd[0], 'custom-spark-submit')示例7: test_resolve_connection_mesos_default_connection
# 需要导入模块: from airflow.contrib.hooks.spark_submit_hook import SparkSubmitHook [as 别名]
# 或者: from airflow.contrib.hooks.spark_submit_hook.SparkSubmitHook import _build_spark_submit_command [as 别名]
def test_resolve_connection_mesos_default_connection(self):
# Given
hook = SparkSubmitHook(conn_id='spark_default_mesos')
# When
connection = hook._resolve_connection()
cmd = hook._build_spark_submit_command(self._spark_job_file)
# Then
dict_cmd = self.cmd_args_to_dict(cmd)
expected_spark_connection = {"master": "mesos://host:5050",
"spark_binary": "spark-submit",
"deploy_mode": None,
"queue": None,
"spark_home": None}
self.assertEqual(connection, expected_spark_connection)
self.assertEqual(dict_cmd["--master"], "mesos://host:5050")示例8: test_resolve_connection_spark_home_set_connection
# 需要导入模块: from airflow.contrib.hooks.spark_submit_hook import SparkSubmitHook [as 别名]
# 或者: from airflow.contrib.hooks.spark_submit_hook.SparkSubmitHook import _build_spark_submit_command [as 别名]
def test_resolve_connection_spark_home_set_connection(self):
# Given
hook = SparkSubmitHook(conn_id='spark_home_set')
# When
connection = hook._resolve_connection()
cmd = hook._build_spark_submit_command(self._spark_job_file)
# Then
expected_spark_connection = {"master": "yarn://yarn-master",
"spark_binary": "spark-submit",
"deploy_mode": None,
"queue": None,
"spark_home": "/opt/myspark",
"namespace": 'default'}
self.assertEqual(connection, expected_spark_connection)
self.assertEqual(cmd[0], '/opt/myspark/bin/spark-submit')示例9: test_resolve_connection_yarn_default
# 需要导入模块: from airflow.contrib.hooks.spark_submit_hook import SparkSubmitHook [as 别名]
# 或者: from airflow.contrib.hooks.spark_submit_hook.SparkSubmitHook import _build_spark_submit_command [as 别名]
def test_resolve_connection_yarn_default(self):
# Given
hook = SparkSubmitHook(conn_id='')
# When
connection = hook._resolve_connection()
cmd = hook._build_spark_submit_command(self._spark_job_file)
# Then
dict_cmd = self.cmd_args_to_dict(cmd)
expected_spark_connection = {"master": "yarn",
"spark_binary": "spark-submit",
"deploy_mode": None,
"queue": None,
"spark_home": None,
"namespace": 'default'}
self.assertEqual(connection, expected_spark_connection)
self.assertEqual(dict_cmd["--master"], "yarn")示例10: test_resolve_connection_spark_yarn_cluster_connection
# 需要导入模块: from airflow.contrib.hooks.spark_submit_hook import SparkSubmitHook [as 别名]
# 或者: from airflow.contrib.hooks.spark_submit_hook.SparkSubmitHook import _build_spark_submit_command [as 别名]
def test_resolve_connection_spark_yarn_cluster_connection(self):
# Given
hook = SparkSubmitHook(conn_id='spark_yarn_cluster')
# When
connection = hook._resolve_connection()
cmd = hook._build_spark_submit_command(self._spark_job_file)
# Then
dict_cmd = self.cmd_args_to_dict(cmd)
expected_spark_connection = {"master": "yarn://yarn-master",
"spark_binary": "spark-submit",
"deploy_mode": "cluster",
"queue": "root.etl",
"spark_home": None}
self.assertEqual(connection, expected_spark_connection)
self.assertEqual(dict_cmd["--master"], "yarn://yarn-master")
self.assertEqual(dict_cmd["--queue"], "root.etl")
self.assertEqual(dict_cmd["--deploy-mode"], "cluster")示例11: test_resolve_connection_spark_k8s_cluster_connection
# 需要导入模块: from airflow.contrib.hooks.spark_submit_hook import SparkSubmitHook [as 别名]
# 或者: from airflow.contrib.hooks.spark_submit_hook.SparkSubmitHook import _build_spark_submit_command [as 别名]
def test_resolve_connection_spark_k8s_cluster_connection(self):
# Given
hook = SparkSubmitHook(conn_id='spark_k8s_cluster')
# When
connection = hook._resolve_connection()
cmd = hook._build_spark_submit_command(self._spark_job_file)
# Then
dict_cmd = self.cmd_args_to_dict(cmd)
expected_spark_connection = {"spark_home": "/opt/spark",
"queue": None,
"spark_binary": "spark-submit",
"master": "k8s://https://k8s-master",
"deploy_mode": "cluster",
"namespace": "mynamespace"}
self.assertEqual(connection, expected_spark_connection)
self.assertEqual(dict_cmd["--master"], "k8s://https://k8s-master")
self.assertEqual(dict_cmd["--deploy-mode"], "cluster")