本文整理汇总了Python中mrjob.inline.InlineMRJobRunner._update_jobconf_for_hadoop_version方法的典型用法代码示例。如果您正苦于以下问题：Python InlineMRJobRunner._update_jobconf_for_hadoop_version方法的具体用法？Python InlineMRJobRunner._update_jobconf_for_hadoop_version怎么用？Python InlineMRJobRunner._update_jobconf_for_hadoop_version使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类mrjob.inline.InlineMRJobRunner的用法示例。
在下文中一共展示了InlineMRJobRunner._update_jobconf_for_hadoop_version方法的2个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: UpdateJobConfForHadoopVersionTestCase
# 需要导入模块: from mrjob.inline import InlineMRJobRunner [as 别名]
# 或者: from mrjob.inline.InlineMRJobRunner import _update_jobconf_for_hadoop_version [as 别名]
class UpdateJobConfForHadoopVersionTestCase(TestCase):
# jobconf with strange mix of Hadoop 1 and Hadoop 2 variables
JOBCONF = {
'foo.bar': 'baz', # unknown jobconf
'mapred.jar': 'a.jar', # Hadoop 1 jobconf
'mapreduce.job.user.name': 'dave', # Hadoop 2 jobconf
}
def setUp(self):
self.runner = InlineMRJobRunner(conf_paths=[])
def updated_and_warnings(self, jobconf, hadoop_version):
jobconf = jobconf.copy()
with no_handlers_for_logger('mrjob.runner'):
stderr = StringIO()
log_to_stream('mrjob.runner', stderr)
self.runner._update_jobconf_for_hadoop_version(
jobconf, hadoop_version)
return jobconf, stderr.getvalue()
def test_no_version(self):
updated, warnings = self.updated_and_warnings(
self.JOBCONF, None)
self.assertEqual(updated, self.JOBCONF)
self.assertEqual(warnings, '')
def test_hadoop_1(self):
updated, warnings = self.updated_and_warnings(
self.JOBCONF, '1.0')
self.assertEqual(updated,
combine_dicts(self.JOBCONF, {'user.name': 'dave'}))
self.assertIn('do not match hadoop version', warnings)
self.assertIn('mapreduce.job.user.name: user.name', warnings)
def test_hadoop_2(self):
updated, warnings = self.updated_and_warnings(
self.JOBCONF, '2.0')
self.assertEqual(updated,
combine_dicts(self.JOBCONF,
{'mapreduce.job.jar': 'a.jar'}))
self.assertIn('do not match hadoop version', warnings)
self.assertIn('mapred.jar: mapreduce.job.jar', warnings)
def test_dont_overwrite(self):
# this jobconf contains two versions of the same variable
jobconf = {'mapred.jar': 'a.jar', 'mapreduce.job.jar': 'b.jar'}
updated, warnings = self.updated_and_warnings(jobconf, '1.0')
self.assertEqual(updated, jobconf)
self.assertEqual(warnings, '')示例2: UpdateJobConfForHadoopVersionTestCase
# 需要导入模块: from mrjob.inline import InlineMRJobRunner [as 别名]
# 或者: from mrjob.inline.InlineMRJobRunner import _update_jobconf_for_hadoop_version [as 别名]
class UpdateJobConfForHadoopVersionTestCase(TestCase):
# jobconf with strange mix of Hadoop 1 and Hadoop 2 variables
JOBCONF = {
"foo.bar": "baz", # unknown jobconf
"mapred.jar": "a.jar", # Hadoop 1 jobconf
"mapreduce.job.user.name": "dave", # Hadoop 2 jobconf
}
def setUp(self):
self.runner = InlineMRJobRunner(conf_paths=[])
def updated_and_warnings(self, jobconf, hadoop_version):
jobconf = jobconf.copy()
with no_handlers_for_logger("mrjob.runner"):
stderr = StringIO()
log_to_stream("mrjob.runner", stderr)
self.runner._update_jobconf_for_hadoop_version(jobconf, hadoop_version)
return jobconf, stderr.getvalue()
def test_no_version(self):
updated, warnings = self.updated_and_warnings(self.JOBCONF, None)
self.assertEqual(updated, self.JOBCONF)
self.assertEqual(warnings, "")
def test_hadoop_1(self):
updated, warnings = self.updated_and_warnings(self.JOBCONF, "1.0")
self.assertEqual(updated, combine_dicts(self.JOBCONF, {"user.name": "dave"}))
self.assertIn("do not match hadoop version", warnings)
self.assertIn("mapreduce.job.user.name: user.name", warnings)
def test_hadoop_2(self):
updated, warnings = self.updated_and_warnings(self.JOBCONF, "2.0")
self.assertEqual(updated, combine_dicts(self.JOBCONF, {"mapreduce.job.jar": "a.jar"}))
self.assertIn("do not match hadoop version", warnings)
self.assertIn("mapred.jar: mapreduce.job.jar", warnings)
def test_dont_overwrite(self):
# this jobconf contains two versions of the same variable
jobconf = {"mapred.jar": "a.jar", "mapreduce.job.jar": "b.jar"}
updated, warnings = self.updated_and_warnings(jobconf, "1.0")
self.assertEqual(updated, jobconf)
self.assertEqual(warnings, "")