本文整理汇总了Python中tests.mr_word_count.MRWordCount.sandbox方法的典型用法代码示例。如果您正苦于以下问题：Python MRWordCount.sandbox方法的具体用法？Python MRWordCount.sandbox怎么用？Python MRWordCount.sandbox使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类tests.mr_word_count.MRWordCount的用法示例。
在下文中一共展示了MRWordCount.sandbox方法的13个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: test_persistent_cluster
# 需要导入模块: from tests.mr_word_count import MRWordCount [as 别名]
# 或者: from tests.mr_word_count.MRWordCount import sandbox [as 别名]
def test_persistent_cluster(self):
mr_job = MRWordCount(["-r", "dataproc", "--max-hours-idle", "0.01"])
mr_job.sandbox()
with mr_job.make_runner() as runner:
runner.run()
self.assertRanIdleTimeoutScriptWith(runner, {"mrjob-max-secs-idle": "36"})示例2: test_input_files_and_setting_number_of_tasks
# 需要导入模块: from tests.mr_word_count import MRWordCount [as 别名]
# 或者: from tests.mr_word_count.MRWordCount import sandbox [as 别名]
def test_input_files_and_setting_number_of_tasks(self):
input_path = os.path.join(self.tmp_dir, 'input')
with open(input_path, 'wb') as input_file:
input_file.write(b'bar\nqux\nfoo\n')
input_gz_path = os.path.join(self.tmp_dir, 'input.gz')
input_gz = gzip.GzipFile(input_gz_path, 'wb')
input_gz.write(b'foo\n')
input_gz.close()
mr_job = MRWordCount(['-r', self.RUNNER,
'--jobconf=mapred.map.tasks=3',
'--jobconf=mapred.reduce.tasks=3',
input_path, input_gz_path])
mr_job.sandbox()
results = []
with mr_job.make_runner() as runner:
runner.run()
for line in runner.stream_output():
key, value = mr_job.parse_output_line(line)
results.append((key, value))
self.assertEqual(runner.counters()[0]['count']['combiners'], 3)
self.assertEqual(sorted(results),
[(input_path, 3), (input_gz_path, 1)])示例3: test_default
# 需要导入模块: from tests.mr_word_count import MRWordCount [as 别名]
# 或者: from tests.mr_word_count.MRWordCount import sandbox [as 别名]
def test_default(self):
mr_job = MRWordCount(["-r", "dataproc"])
mr_job.sandbox()
with mr_job.make_runner() as runner:
runner.run()
self.assertRanIdleTimeoutScriptWith(runner, {"mrjob-max-secs-idle": "360"})示例4: test_persistent_cluster
# 需要导入模块: from tests.mr_word_count import MRWordCount [as 别名]
# 或者: from tests.mr_word_count.MRWordCount import sandbox [as 别名]
def test_persistent_cluster(self):
mr_job = MRWordCount(['-r', 'dataproc', '--max-hours-idle', '0.01'])
mr_job.sandbox()
with mr_job.make_runner() as runner:
runner.run()
self.assertRanIdleTimeoutScriptWith(runner, {
'mrjob-max-secs-idle': '36',
})示例5: test_default
# 需要导入模块: from tests.mr_word_count import MRWordCount [as 别名]
# 或者: from tests.mr_word_count.MRWordCount import sandbox [as 别名]
def test_default(self):
mr_job = MRWordCount(['-r', 'dataproc'])
mr_job.sandbox()
with mr_job.make_runner() as runner:
runner.run()
self.assertRanIdleTimeoutScriptWith(runner, {
'mrjob-max-secs-idle': '360',
})示例6: test_empty
# 需要导入模块: from tests.mr_word_count import MRWordCount [as 别名]
# 或者: from tests.mr_word_count.MRWordCount import sandbox [as 别名]
def test_empty(self):
job = MRWordCount(['-r', 'hadoop'])
job.sandbox()
with job.make_runner() as runner:
runner._add_job_files_for_upload()
args = runner._args_for_streaming_step(0)
self.assertNotIn('-libjars', args)示例7: test_job_passes_in_steps
# 需要导入模块: from tests.mr_word_count import MRWordCount [as 别名]
# 或者: from tests.mr_word_count.MRWordCount import sandbox [as 别名]
def test_job_passes_in_steps(self):
job = MRWordCount()
job.sandbox()
with job.make_runner() as runner:
self.assertTrue(runner._steps)
runner.run()
self.assertFalse(self.log.warning.called)示例8: test_load_steps
# 需要导入模块: from tests.mr_word_count import MRWordCount [as 别名]
# 或者: from tests.mr_word_count.MRWordCount import sandbox [as 别名]
def test_load_steps(self):
job = MRWordCount()
job.sandbox()
with job.make_runner() as runner:
runner._steps = None
runner.run()
self.assertTrue(runner._steps)
self.assertTrue(self.log.warning.called)示例9: test_one_jar
# 需要导入模块: from tests.mr_word_count import MRWordCount [as 别名]
# 或者: from tests.mr_word_count.MRWordCount import sandbox [as 别名]
def test_one_jar(self):
job = MRWordCount([
'-r', 'hadoop',
'--libjar', '/path/to/a.jar',
])
job.sandbox()
with job.make_runner() as runner:
runner._add_job_files_for_upload()
args = runner._args_for_streaming_step(0)
self.assertIn('-libjars', args)
self.assertIn('/path/to/a.jar', args)示例10: test_input_files
# 需要导入模块: from tests.mr_word_count import MRWordCount [as 别名]
# 或者: from tests.mr_word_count.MRWordCount import sandbox [as 别名]
def test_input_files(self):
input_path = os.path.join(self.tmp_dir, 'input')
with open(input_path, 'wb') as input_file:
input_file.write(b'bar\nqux\nfoo\n')
input_gz_path = os.path.join(self.tmp_dir, 'input.gz')
with gzip.GzipFile(input_gz_path, 'wb') as input_gz:
input_gz.write(b'foo\n')
mr_job = MRWordCount(['-r', self.RUNNER,
input_path, input_gz_path])
mr_job.sandbox()
results = []
with mr_job.make_runner() as runner:
runner.run()
results.extend(mr_job.parse_output(runner.cat_output()))
self.assertGreater(runner.counters()[0]['count']['combiners'], 2)
self.assertEqual(sorted(results),
[(input_path, 3), (input_gz_path, 1)])示例11: test_dash_for_stdin
# 需要导入模块: from tests.mr_word_count import MRWordCount [as 别名]
# 或者: from tests.mr_word_count.MRWordCount import sandbox [as 别名]
def test_dash_for_stdin(self):
job = MRWordCount(['-'])
job.sandbox()
with job.make_runner() as runner:
runner.run()示例12: test_stdin_is_fine
# 需要导入模块: from tests.mr_word_count import MRWordCount [as 别名]
# 或者: from tests.mr_word_count.MRWordCount import sandbox [as 别名]
def test_stdin_is_fine(self):
job = MRWordCount()
job.sandbox()
with job.make_runner() as runner:
runner.run()示例13: make_runner
# 需要导入模块: from tests.mr_word_count import MRWordCount [as 别名]
# 或者: from tests.mr_word_count.MRWordCount import sandbox [as 别名]
def make_runner(self, *args):
mr_job = MRWordCount(args)
mr_job.sandbox()
return mr_job.make_runner()