本文整理汇总了Python中mrjob.util.bash_wrap函数的典型用法代码示例。如果您正苦于以下问题：Python bash_wrap函数的具体用法？Python bash_wrap怎么用？Python bash_wrap使用的例子？那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了bash_wrap函数的12个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: _hadoop_streaming_commands
def _hadoop_streaming_commands(self, step_num):
version = self.get_hadoop_version()
# Hadoop streaming stuff
mapper, bash_wrap_mapper = self._render_substep(
step_num, 'mapper')
combiner, bash_wrap_combiner = self._render_substep(
step_num, 'combiner')
reducer, bash_wrap_reducer = self._render_substep(
step_num, 'reducer')
if (combiner is not None and
not supports_combiners_in_hadoop_streaming(version)):
# krazy hack to support combiners on hadoop <0.20
bash_wrap_mapper = True
mapper = "%s | sort | %s" % (mapper, combiner)
# take the combiner away, hadoop will just be confused
combiner = None
bash_wrap_combiner = False
if bash_wrap_mapper:
mapper = bash_wrap(mapper)
if bash_wrap_combiner:
combiner = bash_wrap(combiner)
if bash_wrap_reducer:
reducer = bash_wrap(reducer)
return mapper, combiner, reducer示例2: steps
def steps(self):
steps = []
for step in self.options.steps:
step_kwargs = {}
if "mapper" in step:
step_kwargs["mapper_cmd"] = bash_wrap(step["mapper"])
if "combiner" in step:
step_kwargs["combiner_cmd"] = bash_wrap(step["combiner"])
if "reducer" in step:
step_kwargs["reducer_cmd"] = bash_wrap(step["reducer"])
steps.append(MRStep(**step_kwargs))
return steps示例3: steps
def steps(self):
steps = []
for step in self.options.steps:
step_kwargs = {}
if 'mapper' in step:
step_kwargs['mapper_cmd'] = bash_wrap(step['mapper'])
if 'combiner' in step:
step_kwargs['combiner_cmd'] = bash_wrap(step['combiner'])
if 'reducer' in step:
step_kwargs['reducer_cmd'] = bash_wrap(step['reducer'])
steps.append(self.mr(**step_kwargs))
return steps示例4: _hadoop_streaming_commands
def _hadoop_streaming_commands(self, step_num):
# Hadoop streaming stuff
mapper, bash_wrap_mapper = self._render_substep(step_num, "mapper")
combiner, bash_wrap_combiner = self._render_substep(step_num, "combiner")
reducer, bash_wrap_reducer = self._render_substep(step_num, "reducer")
if bash_wrap_mapper:
mapper = bash_wrap(mapper)
if bash_wrap_combiner:
combiner = bash_wrap(combiner)
if bash_wrap_reducer:
reducer = bash_wrap(reducer)
return mapper, combiner, reducer示例5: test_pre_filter_escaping
def test_pre_filter_escaping(self):
# ESCAPE ALL THE THINGS!!!
self._assert_streaming_step(
{"type": "streaming", "mapper": {"type": "script", "pre_filter": bash_wrap("grep 'anything'")}},
[
"-mapper",
"bash -c 'bash -c '\\''grep"
" '\\''\\'\\'''\\''anything'\\''\\'\\'''\\'''\\'' |"
" python my_job.py --step-num=0 --mapper'",
"-jobconf",
"mapred.reduce.tasks=0",
],
)示例6: _hadoop_streaming_commands
def _hadoop_streaming_commands(self, step_num):
version = self.get_hadoop_version()
# Hadoop streaming stuff
mapper, bash_wrap_mapper = self._render_substep(
step_num, 'mapper')
combiner, bash_wrap_combiner = self._render_substep(
step_num, 'combiner')
reducer, bash_wrap_reducer = self._render_substep(
step_num, 'reducer')
if bash_wrap_mapper:
mapper = bash_wrap(mapper)
if bash_wrap_combiner:
combiner = bash_wrap(combiner)
if bash_wrap_reducer:
reducer = bash_wrap(reducer)
return mapper, combiner, reducer示例7: test_pre_filter_escaping
def test_pre_filter_escaping(self):
# ESCAPE ALL THE THINGS!!!
self._assert_streaming_step(
{
'type': 'streaming',
'mapper': {
'type': 'script',
'pre_filter': bash_wrap("grep 'anything'"),
},
},
['-mapper',
"bash -c 'bash -c '\\''grep"
" '\\''\\'\\'''\\''anything'\\''\\'\\'''\\'''\\'' |"
" python my_job.py --step-num=0 --mapper'",
'-jobconf', 'mapred.reduce.tasks=0'])示例8: test_pre_filter_escaping
def test_pre_filter_escaping(self):
# ESCAPE ALL THE THINGS!!!
self.runner._steps = [
{
'type': 'streaming',
'mapper': {
'type': 'script',
'pre_filter': bash_wrap("grep 'anything'"),
},
},
]
self.assertEqual(
self.runner._args_for_streaming_step(0),
(self.BASIC_HADOOP_ARGS + ['-D', 'mapreduce.job.reduces=0'] +
self.BASIC_JOB_ARGS + [
'-mapper',
"bash -c 'bash -c '\\''grep"
" '\\''\\'\\'''\\''anything'\\''\\'\\'''\\'''\\'' | " +
PYTHON_BIN +
" my_job.py --step-num=0 --mapper'"]))示例9: test_multiple
def test_multiple(self):
data = 'x\nx\nx\nx\nx\nx\n'
mapper_cmd = 'cat -e'
reducer_cmd = bash_wrap('wc -l | tr -Cd "[:digit:]"')
job = CmdJob([
'--runner', 'local',
'--mapper-cmd', mapper_cmd,
'--combiner-cmd', 'uniq',
'--reducer-cmd', reducer_cmd])
job.sandbox(stdin=StringIO(data))
with job.make_runner() as r:
self.assertEqual(
r._get_steps(),
[{
'type': 'streaming',
'mapper': {'type': 'command', 'command': mapper_cmd},
'combiner': {'type': 'command', 'command': 'uniq'},
'reducer': {'type': 'command', 'command': reducer_cmd},
}])
r.run()
self.assertEqual(list(r.stream_output()), ['2'])示例10: test_multiple
def test_multiple(self):
data = b"x\nx\nx\nx\nx\nx\n"
mapper_cmd = "cat -e"
reducer_cmd = bash_wrap('wc -l | tr -Cd "[:digit:]"')
job = CmdJob(
["--runner", "local", "--mapper-cmd", mapper_cmd, "--combiner-cmd", "uniq", "--reducer-cmd", reducer_cmd]
)
job.sandbox(stdin=BytesIO(data))
with job.make_runner() as r:
self.assertEqual(
r._get_steps(),
[
{
"type": "streaming",
"mapper": {"type": "command", "command": mapper_cmd},
"combiner": {"type": "command", "command": "uniq"},
"reducer": {"type": "command", "command": reducer_cmd},
}
],
)
r.run()
self.assertEqual(list(r.stream_output()), [b"2"])示例11: reducer_cmd
def reducer_cmd(self):
return bash_wrap('./wordcount.sh reducer')示例12: mapper_cmd
def mapper_cmd(self):
return bash_wrap('./wordcount.sh mapper')