本文整理汇总了Python中mrjob.setup.parse_setup_cmd函数的典型用法代码示例。如果您正苦于以下问题：Python parse_setup_cmd函数的具体用法？Python parse_setup_cmd怎么用？Python parse_setup_cmd使用的例子？那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了parse_setup_cmd函数的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: test_hash_path_alone
def test_hash_path_alone(self):
self.assertEqual(parse_setup_cmd("foo#bar"), [{"type": "file", "path": "foo", "name": "bar"}])
self.assertEqual(parse_setup_cmd("/dir/foo#bar"), [{"type": "file", "path": "/dir/foo", "name": "bar"}])
self.assertEqual(parse_setup_cmd("foo#bar/"), [{"type": "archive", "path": "foo", "name": "bar"}, "/"])
self.assertEqual(
parse_setup_cmd("/dir/foo#bar/"), [{"type": "archive", "path": "/dir/foo", "name": "bar"}, "/"]
)示例2: test_no_name
def test_no_name(self):
self.assertEqual(
parse_setup_cmd('foo#'),
[{'type': 'file', 'path': 'foo', 'name': None}])
self.assertEqual(
parse_setup_cmd('foo#/'),
[{'type': 'archive', 'path': 'foo', 'name': None}, '/'])示例3: test_archive_hash_path_alone
def test_archive_hash_path_alone(self):
self.assertEqual(
parse_setup_cmd('foo#/'),
[{'type': 'archive', 'path': 'foo', 'name': None}, '/'])
self.assertEqual(
parse_setup_cmd('foo#bar/'),
[{'type': 'archive', 'path': 'foo', 'name': 'bar'}, '/'])
self.assertEqual(
parse_setup_cmd('/dir/foo#bar/'),
[{'type': 'archive', 'path': '/dir/foo', 'name': 'bar'}, '/'])示例4: test_file_hash_path_alone
def test_file_hash_path_alone(self):
self.assertEqual(
parse_setup_cmd('foo#'),
[{'type': 'file', 'path': 'foo', 'name': None}])
self.assertEqual(
parse_setup_cmd('foo#bar'),
[{'type': 'file', 'path': 'foo', 'name': 'bar'}])
self.assertEqual(
parse_setup_cmd('/dir/foo#bar'),
[{'type': 'file', 'path': '/dir/foo', 'name': 'bar'}])示例5: _parse_setup
def _parse_setup(self):
"""Parse the *setup* option with
:py:func:`mrjob.setup.parse_setup_cmd()`.
If *bootstrap_mrjob* and ``self.BOOTSTRAP_MRJOB_IN_SETUP`` are both
true, create mrjob.tar.gz (if it doesn't exist already) and
prepend a setup command that adds it to PYTHONPATH.
Also patch in the deprecated
options *python_archives*, *setup_cmd*, and *setup_script*
as setup commands.
"""
setup = []
# python_archives
for path in self._opts['python_archives']:
path_dict = parse_legacy_hash_path('archive', path)
setup.append(['export PYTHONPATH=', path_dict, ':$PYTHONPATH'])
# setup
for cmd in self._opts['setup']:
setup.append(parse_setup_cmd(cmd))
# setup_cmds
for cmd in self._opts['setup_cmds']:
if not isinstance(cmd, basestring):
cmd = cmd_line(cmd)
setup.append([cmd])
# setup_scripts
for path in self._opts['setup_scripts']:
path_dict = parse_legacy_hash_path('file', path)
setup.append([path_dict])
return setup示例6: test_resolve_path_but_not_name
def test_resolve_path_but_not_name(self):
with patch.dict(os.environ, {'HOME': '/home/foo',
'USER': 'foo',
'BAR': 'bar'}, clear=True):
self.assertEqual(
parse_setup_cmd(r'. ~/tmp/$USER/\$BAR.sh#$USER.sh'),
['. ',
{'path': '/home/foo/tmp/foo/$BAR.sh',
'name': '$USER.sh',
'type': 'file'}])示例7: test_shell_punctuation_after_name
def test_shell_punctuation_after_name(self):
self.assertEqual(
parse_setup_cmd('touch foo#; cat bar#>baz; cat qux#|grep quux'),
['touch ',
{'type': 'file', 'path': 'foo', 'name': None},
'; cat ',
{'type': 'file', 'path': 'bar', 'name': None},
'>baz; cat ',
{'type': 'file', 'path': 'qux', 'name': None},
'|grep quux'])示例8: test_shell_punctuation_after_name
def test_shell_punctuation_after_name(self):
self.assertEqual(
parse_setup_cmd("touch foo#; cat bar#>baz; cat qux#|grep quux"),
[
"touch ",
{"type": "file", "path": "foo", "name": None},
"; cat ",
{"type": "file", "path": "bar", "name": None},
">baz; cat ",
{"type": "file", "path": "qux", "name": None},
"|grep quux",
],
)示例9: _parse_setup
def _parse_setup(self):
"""Parse the *setup* option with
:py:func:`mrjob.setup.parse_setup_cmd()`.
If *bootstrap_mrjob* and ``self.BOOTSTRAP_MRJOB_IN_SETUP`` are both
true, create mrjob.tar.gz (if it doesn't exist already) and
prepend a setup command that adds it to PYTHONPATH.
Also patch in the deprecated
options *python_archives*, *setup_cmd*, and *setup_script*
as setup commands.
"""
setup = []
# python_archives
for path in self._opts["python_archives"]:
path_dict = parse_legacy_hash_path("archive", path)
setup.append(["export PYTHONPATH=", path_dict, ":$PYTHONPATH"])
# setup
for cmd in self._opts["setup"]:
setup.append(parse_setup_cmd(cmd))
# setup_cmds
if self._opts["setup_cmds"]:
log.warning(
"setup_cmds is deprecated since v0.4.2 and will be removed" " in v0.6.0. Consider using setup instead."
)
for cmd in self._opts["setup_cmds"]:
if not isinstance(cmd, string_types):
cmd = cmd_line(cmd)
setup.append([cmd])
# setup_scripts
if self._opts["setup_scripts"]:
log.warning(
"setup_scripts is deprecated since v0.4.2 and will be removed"
" in v0.6.0. Consider using setup instead."
)
for path in self._opts["setup_scripts"]:
path_dict = parse_legacy_hash_path("file", path)
setup.append([path_dict])
return setup示例10: _parse_setup_and_py_files
def _parse_setup_and_py_files(self):
"""Parse the *setup* option with
:py:func:`mrjob.setup.parse_setup_cmd()`, and patch in *py_files*.
"""
setup = []
# py_files
for path in self._opts['py_files']:
# Spark (at least v1.3.1) doesn't work with # and --py-files,
# see #1375
if '#' in path:
raise ValueError("py_files cannot contain '#'")
path_dict = parse_legacy_hash_path('file', path)
setup.append(['export PYTHONPATH=', path_dict, ':$PYTHONPATH'])
# setup
for cmd in self._opts['setup']:
setup.append(parse_setup_cmd(cmd))
return setup示例11: __init__
def __init__(self, **kwargs):
super(MRJobBinRunner, self).__init__(**kwargs)
# where a zip file of the mrjob library is stored locally
self._mrjob_zip_path = None
# we'll create the setup wrapper scripts later
self._setup_wrapper_script_path = None
self._manifest_setup_script_path = None
self._spark_python_wrapper_path = None
# self._setup is a list of shell commands with path dicts
# interleaved; see mrjob.setup.parse_setup_cmd() for details
self._setup = [parse_setup_cmd(cmd) for cmd in self._opts['setup']]
if self._setup and self._has_pyspark_steps() and not (
self._spark_setup_is_supported()):
log.warning("setup commands aren't supported on Spark master %r" %
self._spark_master())
for cmd in self._setup:
for token in cmd:
if isinstance(token, dict):
# convert dir archives tokens to archives
if token['type'] == 'dir':
# feed the archive's path to self._working_dir_mgr
token['path'] = self._dir_archive_path(token['path'])
token['type'] = 'archive'
self._working_dir_mgr.add(**token)
# --py-files on Spark doesn't allow '#' (see #1375)
if any('#' in path for path in self._opts['py_files']):
raise ValueError("py_files cannot contain '#'")
# Keep track of where the spark-submit binary is
self._spark_submit_bin = self._opts['spark_submit_bin']示例12: test_dir_hash_path_alone
def test_dir_hash_path_alone(self):
self.assertEqual(
parse_setup_cmd('foo/#'),
[{'type': 'dir', 'path': 'foo', 'name': None}, '/'])
self.assertEqual(
parse_setup_cmd('foo/#/'),
[{'type': 'dir', 'path': 'foo', 'name': None}, '/'])
self.assertEqual(
parse_setup_cmd('foo/#bar'),
[{'type': 'dir', 'path': 'foo', 'name': 'bar'}, '/'])
self.assertEqual(
parse_setup_cmd('foo/#bar/'),
[{'type': 'dir', 'path': 'foo', 'name': 'bar'}, '/'])
self.assertEqual(
parse_setup_cmd('/dir/foo/#bar'),
[{'type': 'dir', 'path': '/dir/foo', 'name': 'bar'}, '/'])
self.assertEqual(
parse_setup_cmd('/dir/foo/#bar/'),
[{'type': 'dir', 'path': '/dir/foo', 'name': 'bar'}, '/'])示例13: test_start_path_after_equals
def test_start_path_after_equals(self):
self.assertEqual(
parse_setup_cmd("export PYTHONPATH=foo.egg#"),
["export PYTHONPATH=", {"type": "file", "path": "foo.egg", "name": None}],
)示例14: test_start_path_after_colon
def test_start_path_after_colon(self):
self.assertEqual(
parse_setup_cmd("export PYTHONPATH=$PYTHONPATH:foo.tar.gz#/"),
["export PYTHONPATH=$PYTHONPATH:", {"type": "archive", "path": "foo.tar.gz", "name": None}, "/"],
)示例15: test_colon_after_name
def test_colon_after_name(self):
self.assertEqual(
parse_setup_cmd("echo foo.egg#:$PYTHONPATH"),
["echo ", {"type": "file", "path": "foo.egg", "name": None}, ":$PYTHONPATH"],
)