本文整理汇总了Python中mrjob.inline.InlineMRJobRunner类的典型用法代码示例。如果您正苦于以下问题：Python InlineMRJobRunner类的具体用法？Python InlineMRJobRunner怎么用？Python InlineMRJobRunner使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了InlineMRJobRunner类的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: test_owner_and_label_kwargs
def test_owner_and_label_kwargs(self):
runner = InlineMRJobRunner(conf_paths=[],
owner='ads', label='ads_chain')
match = _JOB_KEY_RE.match(runner.get_job_key())
self.assertEqual(match.group(1), 'ads_chain')
self.assertEqual(match.group(2), 'ads')示例2: test_auto_owner
def test_auto_owner(self):
os.environ['USER'] = 'mcp'
runner = InlineMRJobRunner(conf_paths=[])
match = _JOB_KEY_RE.match(runner.get_job_key())
self.assertEqual(match.group(1), 'no_script')
self.assertEqual(match.group(2), 'mcp')示例3: test_owner_and_label_kwargs
def test_owner_and_label_kwargs(self):
runner = InlineMRJobRunner(conf_path=False,
owner='ads', label='ads_chain')
match = JOB_NAME_RE.match(runner.get_job_name())
self.assertEqual(match.group(1), 'ads_chain')
self.assertEqual(match.group(2), 'ads')示例4: test_empty_no_user
def test_empty_no_user(self):
self.getuser_should_fail = True
runner = InlineMRJobRunner(conf_paths=[])
match = JOB_NAME_RE.match(runner.get_job_name())
self.assertEqual(match.group(1), "no_script")
self.assertEqual(match.group(2), "no_user")示例5: test_stream_output
def test_stream_output(self):
a_dir_path = os.path.join(self.tmp_dir, 'a')
b_dir_path = os.path.join(self.tmp_dir, 'b')
l_dir_path = os.path.join(self.tmp_dir, '_logs')
os.mkdir(a_dir_path)
os.mkdir(b_dir_path)
os.mkdir(l_dir_path)
a_file_path = os.path.join(a_dir_path, 'part-00000')
b_file_path = os.path.join(b_dir_path, 'part-00001')
c_file_path = os.path.join(self.tmp_dir, 'part-00002')
x_file_path = os.path.join(l_dir_path, 'log.xml')
y_file_path = os.path.join(self.tmp_dir, '_SUCCESS')
with open(a_file_path, 'w') as f:
f.write('A')
with open(b_file_path, 'w') as f:
f.write('B')
with open(c_file_path, 'w') as f:
f.write('C')
with open(x_file_path, 'w') as f:
f.write('<XML XML XML/>')
with open(y_file_path, 'w') as f:
f.write('I win')
runner = InlineMRJobRunner(conf_paths=[], output_dir=self.tmp_dir)
self.assertEqual(sorted(runner.stream_output()),
[b'A', b'B', b'C'])示例6: test_stream_output
def test_stream_output(self):
a_dir_path = os.path.join(self.tmp_dir, "a")
b_dir_path = os.path.join(self.tmp_dir, "b")
l_dir_path = os.path.join(self.tmp_dir, "_logs")
os.mkdir(a_dir_path)
os.mkdir(b_dir_path)
os.mkdir(l_dir_path)
a_file_path = os.path.join(a_dir_path, "part-00000")
b_file_path = os.path.join(b_dir_path, "part-00001")
c_file_path = os.path.join(self.tmp_dir, "part-00002")
x_file_path = os.path.join(l_dir_path, "log.xml")
y_file_path = os.path.join(self.tmp_dir, "_SUCCESS")
with open(a_file_path, "w") as f:
f.write("A")
with open(b_file_path, "w") as f:
f.write("B")
with open(c_file_path, "w") as f:
f.write("C")
with open(x_file_path, "w") as f:
f.write("<XML XML XML/>")
with open(y_file_path, "w") as f:
f.write("I win")
runner = InlineMRJobRunner(conf_paths=[], output_dir=self.tmp_dir)
self.assertEqual(sorted(runner.stream_output()), ["A", "B", "C"])示例7: test_auto_owner
def test_auto_owner(self):
os.environ['USER'] = 'mcp'
runner = InlineMRJobRunner(conf_path=False)
match = JOB_NAME_RE.match(runner.get_job_name())
self.assertEqual(match.group(1), 'no_script')
self.assertEqual(match.group(2), 'mcp')示例8: test_getattr_forward
def test_getattr_forward(self):
with no_handlers_for_logger():
r = InlineMRJobRunner(conf_path=False)
store = r._opts
self.assertIsInstance(store, InlineRunnerOptionStore)
a = r.get_default_opts()
self.assertEqual(a, store.default_options())示例9: test_doesnt_actually_create_archive
def test_doesnt_actually_create_archive(self):
archive_dir = self.makedirs('archive')
runner = InlineMRJobRunner()
archive_path = runner._dir_archive_path(archive_dir)
self.assertFalse(os.path.exists(archive_path))示例10: test_auto_owner
def test_auto_owner(self):
os.environ["USER"] = "mcp"
runner = InlineMRJobRunner(conf_paths=[])
match = JOB_NAME_RE.match(runner.get_job_name())
self.assertEqual(match.group(1), "no_script")
self.assertEqual(match.group(2), "mcp")示例11: test_empty_no_user
def test_empty_no_user(self):
self.getuser_should_fail = True
runner = InlineMRJobRunner(conf_paths=[])
match = _JOB_KEY_RE.match(runner.get_job_key())
self.assertEqual(match.group(1), 'no_script')
self.assertEqual(match.group(2), 'no_user')示例12: test_trailing_slash
def test_trailing_slash(self):
archive_dir = self.makedirs('archive') + os.sep
runner = InlineMRJobRunner()
archive_path = runner._dir_archive_path(archive_dir)
self.assertEqual(os.path.basename(archive_path), 'archive.tar.gz')示例13: test_same_dir_twice
def test_same_dir_twice(self):
archive_dir = self.makedirs('archive')
runner = InlineMRJobRunner()
archive_path_1 = runner._dir_archive_path(archive_dir)
archive_path_2 = runner._dir_archive_path(archive_dir)
self.assertEqual(os.path.basename(archive_path_1), 'archive.tar.gz')
self.assertEqual(archive_path_1, archive_path_2)示例14: UpdateJobConfForHadoopVersionTestCase
class UpdateJobConfForHadoopVersionTestCase(TestCase):
# jobconf with strange mix of Hadoop 1 and Hadoop 2 variables
JOBCONF = {
'foo.bar': 'baz', # unknown jobconf
'mapred.jar': 'a.jar', # Hadoop 1 jobconf
'mapreduce.job.user.name': 'dave', # Hadoop 2 jobconf
}
def setUp(self):
self.runner = InlineMRJobRunner(conf_paths=[])
def updated_and_warnings(self, jobconf, hadoop_version):
jobconf = jobconf.copy()
with no_handlers_for_logger('mrjob.runner'):
stderr = StringIO()
log_to_stream('mrjob.runner', stderr)
self.runner._update_jobconf_for_hadoop_version(
jobconf, hadoop_version)
return jobconf, stderr.getvalue()
def test_no_version(self):
updated, warnings = self.updated_and_warnings(
self.JOBCONF, None)
self.assertEqual(updated, self.JOBCONF)
self.assertEqual(warnings, '')
def test_hadoop_1(self):
updated, warnings = self.updated_and_warnings(
self.JOBCONF, '1.0')
self.assertEqual(updated,
combine_dicts(self.JOBCONF, {'user.name': 'dave'}))
self.assertIn('do not match hadoop version', warnings)
self.assertIn('mapreduce.job.user.name: user.name', warnings)
def test_hadoop_2(self):
updated, warnings = self.updated_and_warnings(
self.JOBCONF, '2.0')
self.assertEqual(updated,
combine_dicts(self.JOBCONF,
{'mapreduce.job.jar': 'a.jar'}))
self.assertIn('do not match hadoop version', warnings)
self.assertIn('mapred.jar: mapreduce.job.jar', warnings)
def test_dont_overwrite(self):
# this jobconf contains two versions of the same variable
jobconf = {'mapred.jar': 'a.jar', 'mapreduce.job.jar': 'b.jar'}
updated, warnings = self.updated_and_warnings(jobconf, '1.0')
self.assertEqual(updated, jobconf)
self.assertEqual(warnings, '')示例15: test_mrjob_zip_compiles
def test_mrjob_zip_compiles(self):
runner = InlineMRJobRunner()
with no_handlers_for_logger('mrjob.runner'):
mrjob_zip = runner._create_mrjob_zip()
ZipFile(mrjob_zip).extractall(self.tmp_dir)
self.assertTrue(
compileall.compile_dir(os.path.join(self.tmp_dir, 'mrjob'),
quiet=1))