当前位置: 首页>>代码示例>>Python>>正文


Python inline.InlineMRJobRunner类代码示例

本文整理汇总了Python中mrjob.inline.InlineMRJobRunner的典型用法代码示例。如果您正苦于以下问题:Python InlineMRJobRunner类的具体用法?Python InlineMRJobRunner怎么用?Python InlineMRJobRunner使用的例子?那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。


在下文中一共展示了InlineMRJobRunner类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。

示例1: test_owner_and_label_kwargs

    def test_owner_and_label_kwargs(self):
        runner = InlineMRJobRunner(conf_paths=[],
                                   owner='ads', label='ads_chain')
        match = _JOB_KEY_RE.match(runner.get_job_key())

        self.assertEqual(match.group(1), 'ads_chain')
        self.assertEqual(match.group(2), 'ads')
开发者ID:anirudhreddy92,项目名称:mrjob,代码行数:7,代码来源:test_runner.py

示例2: test_auto_owner

    def test_auto_owner(self):
        os.environ['USER'] = 'mcp'
        runner = InlineMRJobRunner(conf_paths=[])
        match = _JOB_KEY_RE.match(runner.get_job_key())

        self.assertEqual(match.group(1), 'no_script')
        self.assertEqual(match.group(2), 'mcp')
开发者ID:anirudhreddy92,项目名称:mrjob,代码行数:7,代码来源:test_runner.py

示例3: test_owner_and_label_kwargs

    def test_owner_and_label_kwargs(self):
        runner = InlineMRJobRunner(conf_path=False,
                                  owner='ads', label='ads_chain')
        match = JOB_NAME_RE.match(runner.get_job_name())

        self.assertEqual(match.group(1), 'ads_chain')
        self.assertEqual(match.group(2), 'ads')
开发者ID:BrandonHaynes,项目名称:mrjob,代码行数:7,代码来源:test_runner.py

示例4: test_empty_no_user

    def test_empty_no_user(self):
        self.getuser_should_fail = True
        runner = InlineMRJobRunner(conf_paths=[])
        match = JOB_NAME_RE.match(runner.get_job_name())

        self.assertEqual(match.group(1), "no_script")
        self.assertEqual(match.group(2), "no_user")
开发者ID:pyzen,项目名称:mrjob,代码行数:7,代码来源:test_runner.py

示例5: test_stream_output

    def test_stream_output(self):
        a_dir_path = os.path.join(self.tmp_dir, 'a')
        b_dir_path = os.path.join(self.tmp_dir, 'b')
        l_dir_path = os.path.join(self.tmp_dir, '_logs')
        os.mkdir(a_dir_path)
        os.mkdir(b_dir_path)
        os.mkdir(l_dir_path)

        a_file_path = os.path.join(a_dir_path, 'part-00000')
        b_file_path = os.path.join(b_dir_path, 'part-00001')
        c_file_path = os.path.join(self.tmp_dir, 'part-00002')
        x_file_path = os.path.join(l_dir_path, 'log.xml')
        y_file_path = os.path.join(self.tmp_dir, '_SUCCESS')

        with open(a_file_path, 'w') as f:
            f.write('A')

        with open(b_file_path, 'w') as f:
            f.write('B')

        with open(c_file_path, 'w') as f:
            f.write('C')

        with open(x_file_path, 'w') as f:
            f.write('<XML XML XML/>')

        with open(y_file_path, 'w') as f:
            f.write('I win')

        runner = InlineMRJobRunner(conf_paths=[], output_dir=self.tmp_dir)
        self.assertEqual(sorted(runner.stream_output()),
                         [b'A', b'B', b'C'])
开发者ID:anirudhreddy92,项目名称:mrjob,代码行数:32,代码来源:test_runner.py

示例6: test_stream_output

    def test_stream_output(self):
        a_dir_path = os.path.join(self.tmp_dir, "a")
        b_dir_path = os.path.join(self.tmp_dir, "b")
        l_dir_path = os.path.join(self.tmp_dir, "_logs")
        os.mkdir(a_dir_path)
        os.mkdir(b_dir_path)
        os.mkdir(l_dir_path)

        a_file_path = os.path.join(a_dir_path, "part-00000")
        b_file_path = os.path.join(b_dir_path, "part-00001")
        c_file_path = os.path.join(self.tmp_dir, "part-00002")
        x_file_path = os.path.join(l_dir_path, "log.xml")
        y_file_path = os.path.join(self.tmp_dir, "_SUCCESS")

        with open(a_file_path, "w") as f:
            f.write("A")

        with open(b_file_path, "w") as f:
            f.write("B")

        with open(c_file_path, "w") as f:
            f.write("C")

        with open(x_file_path, "w") as f:
            f.write("<XML XML XML/>")

        with open(y_file_path, "w") as f:
            f.write("I win")

        runner = InlineMRJobRunner(conf_paths=[], output_dir=self.tmp_dir)
        self.assertEqual(sorted(runner.stream_output()), ["A", "B", "C"])
开发者ID:pyzen,项目名称:mrjob,代码行数:31,代码来源:test_runner.py

示例7: test_auto_owner

    def test_auto_owner(self):
        os.environ['USER'] = 'mcp'
        runner = InlineMRJobRunner(conf_path=False)
        match = JOB_NAME_RE.match(runner.get_job_name())

        self.assertEqual(match.group(1), 'no_script')
        self.assertEqual(match.group(2), 'mcp')
开发者ID:BrandonHaynes,项目名称:mrjob,代码行数:7,代码来源:test_runner.py

示例8: test_getattr_forward

 def test_getattr_forward(self):
     with no_handlers_for_logger():
         r = InlineMRJobRunner(conf_path=False)
     store = r._opts
     self.assertIsInstance(store, InlineRunnerOptionStore)
     a = r.get_default_opts()
     self.assertEqual(a, store.default_options())
开发者ID:icio,项目名称:mrjob,代码行数:7,代码来源:test_conf.py

示例9: test_doesnt_actually_create_archive

    def test_doesnt_actually_create_archive(self):
        archive_dir = self.makedirs('archive')

        runner = InlineMRJobRunner()
        archive_path = runner._dir_archive_path(archive_dir)

        self.assertFalse(os.path.exists(archive_path))
开发者ID:okomestudio,项目名称:mrjob,代码行数:7,代码来源:test_runner.py

示例10: test_auto_owner

    def test_auto_owner(self):
        os.environ["USER"] = "mcp"
        runner = InlineMRJobRunner(conf_paths=[])
        match = JOB_NAME_RE.match(runner.get_job_name())

        self.assertEqual(match.group(1), "no_script")
        self.assertEqual(match.group(2), "mcp")
开发者ID:pyzen,项目名称:mrjob,代码行数:7,代码来源:test_runner.py

示例11: test_empty_no_user

    def test_empty_no_user(self):
        self.getuser_should_fail = True
        runner = InlineMRJobRunner(conf_paths=[])
        match = _JOB_KEY_RE.match(runner.get_job_key())

        self.assertEqual(match.group(1), 'no_script')
        self.assertEqual(match.group(2), 'no_user')
开发者ID:anirudhreddy92,项目名称:mrjob,代码行数:7,代码来源:test_runner.py

示例12: test_trailing_slash

    def test_trailing_slash(self):
        archive_dir = self.makedirs('archive') + os.sep

        runner = InlineMRJobRunner()
        archive_path = runner._dir_archive_path(archive_dir)

        self.assertEqual(os.path.basename(archive_path), 'archive.tar.gz')
开发者ID:okomestudio,项目名称:mrjob,代码行数:7,代码来源:test_runner.py

示例13: test_same_dir_twice

    def test_same_dir_twice(self):
        archive_dir = self.makedirs('archive')

        runner = InlineMRJobRunner()
        archive_path_1 = runner._dir_archive_path(archive_dir)
        archive_path_2 = runner._dir_archive_path(archive_dir)

        self.assertEqual(os.path.basename(archive_path_1), 'archive.tar.gz')
        self.assertEqual(archive_path_1, archive_path_2)
开发者ID:okomestudio,项目名称:mrjob,代码行数:9,代码来源:test_runner.py

示例14: UpdateJobConfForHadoopVersionTestCase

class UpdateJobConfForHadoopVersionTestCase(TestCase):

    # jobconf with strange mix of Hadoop 1 and Hadoop 2 variables
    JOBCONF = {
        'foo.bar': 'baz',                   # unknown jobconf
        'mapred.jar': 'a.jar',              # Hadoop 1 jobconf
        'mapreduce.job.user.name': 'dave',  # Hadoop 2 jobconf
    }

    def setUp(self):
        self.runner = InlineMRJobRunner(conf_paths=[])

    def updated_and_warnings(self, jobconf, hadoop_version):
        jobconf = jobconf.copy()
        with no_handlers_for_logger('mrjob.runner'):
            stderr = StringIO()
            log_to_stream('mrjob.runner', stderr)
            self.runner._update_jobconf_for_hadoop_version(
                jobconf, hadoop_version)

        return jobconf, stderr.getvalue()

    def test_no_version(self):
        updated, warnings = self.updated_and_warnings(
            self.JOBCONF, None)

        self.assertEqual(updated, self.JOBCONF)
        self.assertEqual(warnings, '')

    def test_hadoop_1(self):
        updated, warnings = self.updated_and_warnings(
            self.JOBCONF, '1.0')

        self.assertEqual(updated,
                         combine_dicts(self.JOBCONF, {'user.name': 'dave'}))
        self.assertIn('do not match hadoop version', warnings)
        self.assertIn('mapreduce.job.user.name: user.name', warnings)

    def test_hadoop_2(self):
        updated, warnings = self.updated_and_warnings(
            self.JOBCONF, '2.0')

        self.assertEqual(updated,
                         combine_dicts(self.JOBCONF,
                                       {'mapreduce.job.jar': 'a.jar'}))
        self.assertIn('do not match hadoop version', warnings)
        self.assertIn('mapred.jar: mapreduce.job.jar', warnings)

    def test_dont_overwrite(self):
        # this jobconf contains two versions of the same variable
        jobconf = {'mapred.jar': 'a.jar', 'mapreduce.job.jar': 'b.jar'}

        updated, warnings = self.updated_and_warnings(jobconf, '1.0')

        self.assertEqual(updated, jobconf)
        self.assertEqual(warnings, '')
开发者ID:Milkigit,项目名称:mrjob,代码行数:56,代码来源:test_runner.py

示例15: test_mrjob_zip_compiles

    def test_mrjob_zip_compiles(self):
        runner = InlineMRJobRunner()
        with no_handlers_for_logger('mrjob.runner'):
            mrjob_zip = runner._create_mrjob_zip()

        ZipFile(mrjob_zip).extractall(self.tmp_dir)

        self.assertTrue(
            compileall.compile_dir(os.path.join(self.tmp_dir, 'mrjob'),
                                   quiet=1))
开发者ID:okomestudio,项目名称:mrjob,代码行数:10,代码来源:test_runner.py


注:本文中的mrjob.inline.InlineMRJobRunner类示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。