本文整理汇总了Python中tests.mr_word_count.MRWordCount类的典型用法代码示例。如果您正苦于以下问题：Python MRWordCount类的具体用法？Python MRWordCount怎么用？Python MRWordCount使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了MRWordCount类的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: test_default
def test_default(self):
mr_job = MRWordCount(["-r", "dataproc"])
mr_job.sandbox()
with mr_job.make_runner() as runner:
runner.run()
self.assertRanIdleTimeoutScriptWith(runner, {"mrjob-max-secs-idle": "360"})示例2: test_partitioner
def test_partitioner(self):
partitioner = 'org.apache.hadoop.mapreduce.Partitioner'
job = MRWordCount(['--partitioner', partitioner])
with job.make_runner() as runner:
self.assertEqual(runner._hadoop_args_for_step(0),
['-partitioner', partitioner])示例3: test_persistent_cluster
def test_persistent_cluster(self):
mr_job = MRWordCount(["-r", "dataproc", "--max-hours-idle", "0.01"])
mr_job.sandbox()
with mr_job.make_runner() as runner:
runner.run()
self.assertRanIdleTimeoutScriptWith(runner, {"mrjob-max-secs-idle": "36"})示例4: test_input_files_and_setting_number_of_tasks
def test_input_files_and_setting_number_of_tasks(self):
input_path = os.path.join(self.tmp_dir, 'input')
with open(input_path, 'wb') as input_file:
input_file.write(b'bar\nqux\nfoo\n')
input_gz_path = os.path.join(self.tmp_dir, 'input.gz')
input_gz = gzip.GzipFile(input_gz_path, 'wb')
input_gz.write(b'foo\n')
input_gz.close()
mr_job = MRWordCount(['-r', self.RUNNER,
'--jobconf=mapred.map.tasks=3',
'--jobconf=mapred.reduce.tasks=3',
input_path, input_gz_path])
mr_job.sandbox()
results = []
with mr_job.make_runner() as runner:
runner.run()
for line in runner.stream_output():
key, value = mr_job.parse_output_line(line)
results.append((key, value))
self.assertEqual(runner.counters()[0]['count']['combiners'], 3)
self.assertEqual(sorted(results),
[(input_path, 3), (input_gz_path, 1)])示例5: test_empty_jobconf_values
def test_empty_jobconf_values(self):
# value of None means to omit that jobconf
job = MRWordCount()
# no way to pass in None from the command line
job.JOBCONF = {"foo": "", "bar": None}
with job.make_runner() as runner:
self.assertEqual(runner._hadoop_args_for_step(0), ["-D", "foo="])示例6: test_configuration_translation
def test_configuration_translation(self):
job = MRWordCount(["--jobconf", "mapred.jobtracker.maxtasks.per.job=1", "--hadoop-version", "0.21"])
with job.make_runner() as runner:
with no_handlers_for_logger("mrjob.compat"):
self.assertEqual(
runner._hadoop_args_for_step(0),
["-D", "mapred.jobtracker.maxtasks.per.job=1", "-D", "mapreduce.jobtracker.maxtasks.perjob=1"],
)示例7: test_persistent_cluster
def test_persistent_cluster(self):
mr_job = MRWordCount(['-r', 'dataproc', '--max-hours-idle', '0.01'])
mr_job.sandbox()
with mr_job.make_runner() as runner:
runner.run()
self.assertRanIdleTimeoutScriptWith(runner, {
'mrjob-max-secs-idle': '36',
})示例8: test_default
def test_default(self):
mr_job = MRWordCount(['-r', 'dataproc'])
mr_job.sandbox()
with mr_job.make_runner() as runner:
runner.run()
self.assertRanIdleTimeoutScriptWith(runner, {
'mrjob-max-secs-idle': '360',
})示例9: test_environment_variables_version_agnostic
def test_environment_variables_version_agnostic(self):
job = MRWordCount(['-r', 'local'])
with job.make_runner() as runner:
simulated_jobconf = runner._simulate_jobconf_for_step(
0, 'mapper', 0, '/tmp/foo')
self.assertIn(
'mapred.cache.localArchives', simulated_jobconf)
self.assertIn(
'mapreduce.job.cache.local.archives', simulated_jobconf)示例10: test_empty
def test_empty(self):
job = MRWordCount(['-r', 'hadoop'])
job.sandbox()
with job.make_runner() as runner:
runner._add_job_files_for_upload()
args = runner._args_for_streaming_step(0)
self.assertNotIn('-libjars', args)示例11: test_environment_variables_hadoop_2
def test_environment_variables_hadoop_2(self):
job = MRWordCount(['-r', 'local', '--hadoop-version', '2.7.2'])
with job.make_runner() as runner:
simulated_jobconf = runner._simulate_jobconf_for_step(
'mapper', 0, 0)
self.assertIn(
'mapreduce.job.cache.local.archives', simulated_jobconf)
self.assertNotIn(
'mapred.cache.localArchives', simulated_jobconf)示例12: test_job_passes_in_steps
def test_job_passes_in_steps(self):
job = MRWordCount()
job.sandbox()
with job.make_runner() as runner:
self.assertTrue(runner._steps)
runner.run()
self.assertFalse(self.log.warning.called)示例13: test_disable_check_input_paths
def test_disable_check_input_paths(self):
missing_data = os.path.join(self.tmp_dir, 'data')
job = MRWordCount(['--no-check-input-paths', missing_data])
self.start(patch('mrjob.inline.InlineMRJobRunner._run',
side_effect=StopIteration))
with job.make_runner() as runner:
self.assertRaises(StopIteration, runner.run)示例14: test_configuration_translation
def test_configuration_translation(self):
job = MRWordCount(["--jobconf", "mapred.jobtracker.maxtasks.per.job=1"])
with job.make_runner() as runner:
with no_handlers_for_logger("mrjob.runner"):
with patch.object(runner, "get_hadoop_version", return_value="2.7.1"):
self.assertEqual(
runner._hadoop_args_for_step(0),
["-D", "mapred.jobtracker.maxtasks.per.job=1", "-D", "mapreduce.jobtracker.maxtasks.perjob=1"],
)示例15: test_cmdenv
def test_cmdenv(self):
job = MRWordCount(['--cmdenv', 'FOO=bar',
'--cmdenv', 'BAZ=qux',
'--cmdenv', 'BAX=Arnold'])
with job.make_runner() as runner:
self.assertEqual(runner._hadoop_args_for_step(0),
['-cmdenv', 'BAX=Arnold',
'-cmdenv', 'BAZ=qux',
'-cmdenv', 'FOO=bar',
])