当前位置: 首页>>代码示例>>Python>>正文


Python InlineMRJobRunner._update_jobconf_for_hadoop_version方法代码示例

本文整理汇总了Python中mrjob.inline.InlineMRJobRunner._update_jobconf_for_hadoop_version方法的典型用法代码示例。如果您正苦于以下问题:Python InlineMRJobRunner._update_jobconf_for_hadoop_version方法的具体用法?Python InlineMRJobRunner._update_jobconf_for_hadoop_version怎么用?Python InlineMRJobRunner._update_jobconf_for_hadoop_version使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在mrjob.inline.InlineMRJobRunner的用法示例。


在下文中一共展示了InlineMRJobRunner._update_jobconf_for_hadoop_version方法的2个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。

示例1: UpdateJobConfForHadoopVersionTestCase

# 需要导入模块: from mrjob.inline import InlineMRJobRunner [as 别名]
# 或者: from mrjob.inline.InlineMRJobRunner import _update_jobconf_for_hadoop_version [as 别名]
class UpdateJobConfForHadoopVersionTestCase(TestCase):

    # jobconf with strange mix of Hadoop 1 and Hadoop 2 variables
    JOBCONF = {
        'foo.bar': 'baz',                   # unknown jobconf
        'mapred.jar': 'a.jar',              # Hadoop 1 jobconf
        'mapreduce.job.user.name': 'dave',  # Hadoop 2 jobconf
    }

    def setUp(self):
        self.runner = InlineMRJobRunner(conf_paths=[])

    def updated_and_warnings(self, jobconf, hadoop_version):
        jobconf = jobconf.copy()
        with no_handlers_for_logger('mrjob.runner'):
            stderr = StringIO()
            log_to_stream('mrjob.runner', stderr)
            self.runner._update_jobconf_for_hadoop_version(
                jobconf, hadoop_version)

        return jobconf, stderr.getvalue()

    def test_no_version(self):
        updated, warnings = self.updated_and_warnings(
            self.JOBCONF, None)

        self.assertEqual(updated, self.JOBCONF)
        self.assertEqual(warnings, '')

    def test_hadoop_1(self):
        updated, warnings = self.updated_and_warnings(
            self.JOBCONF, '1.0')

        self.assertEqual(updated,
                         combine_dicts(self.JOBCONF, {'user.name': 'dave'}))
        self.assertIn('do not match hadoop version', warnings)
        self.assertIn('mapreduce.job.user.name: user.name', warnings)

    def test_hadoop_2(self):
        updated, warnings = self.updated_and_warnings(
            self.JOBCONF, '2.0')

        self.assertEqual(updated,
                         combine_dicts(self.JOBCONF,
                                       {'mapreduce.job.jar': 'a.jar'}))
        self.assertIn('do not match hadoop version', warnings)
        self.assertIn('mapred.jar: mapreduce.job.jar', warnings)

    def test_dont_overwrite(self):
        # this jobconf contains two versions of the same variable
        jobconf = {'mapred.jar': 'a.jar', 'mapreduce.job.jar': 'b.jar'}

        updated, warnings = self.updated_and_warnings(jobconf, '1.0')

        self.assertEqual(updated, jobconf)
        self.assertEqual(warnings, '')
开发者ID:Milkigit,项目名称:mrjob,代码行数:58,代码来源:test_runner.py

示例2: UpdateJobConfForHadoopVersionTestCase

# 需要导入模块: from mrjob.inline import InlineMRJobRunner [as 别名]
# 或者: from mrjob.inline.InlineMRJobRunner import _update_jobconf_for_hadoop_version [as 别名]
class UpdateJobConfForHadoopVersionTestCase(TestCase):

    # jobconf with strange mix of Hadoop 1 and Hadoop 2 variables
    JOBCONF = {
        "foo.bar": "baz",  # unknown jobconf
        "mapred.jar": "a.jar",  # Hadoop 1 jobconf
        "mapreduce.job.user.name": "dave",  # Hadoop 2 jobconf
    }

    def setUp(self):
        self.runner = InlineMRJobRunner(conf_paths=[])

    def updated_and_warnings(self, jobconf, hadoop_version):
        jobconf = jobconf.copy()
        with no_handlers_for_logger("mrjob.runner"):
            stderr = StringIO()
            log_to_stream("mrjob.runner", stderr)
            self.runner._update_jobconf_for_hadoop_version(jobconf, hadoop_version)

        return jobconf, stderr.getvalue()

    def test_no_version(self):
        updated, warnings = self.updated_and_warnings(self.JOBCONF, None)

        self.assertEqual(updated, self.JOBCONF)
        self.assertEqual(warnings, "")

    def test_hadoop_1(self):
        updated, warnings = self.updated_and_warnings(self.JOBCONF, "1.0")

        self.assertEqual(updated, combine_dicts(self.JOBCONF, {"user.name": "dave"}))
        self.assertIn("do not match hadoop version", warnings)
        self.assertIn("mapreduce.job.user.name: user.name", warnings)

    def test_hadoop_2(self):
        updated, warnings = self.updated_and_warnings(self.JOBCONF, "2.0")

        self.assertEqual(updated, combine_dicts(self.JOBCONF, {"mapreduce.job.jar": "a.jar"}))
        self.assertIn("do not match hadoop version", warnings)
        self.assertIn("mapred.jar: mapreduce.job.jar", warnings)

    def test_dont_overwrite(self):
        # this jobconf contains two versions of the same variable
        jobconf = {"mapred.jar": "a.jar", "mapreduce.job.jar": "b.jar"}

        updated, warnings = self.updated_and_warnings(jobconf, "1.0")

        self.assertEqual(updated, jobconf)
        self.assertEqual(warnings, "")
开发者ID:irskep,项目名称:mrjob,代码行数:51,代码来源:test_runner.py


注:本文中的mrjob.inline.InlineMRJobRunner._update_jobconf_for_hadoop_version方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。