当前位置: 首页>>代码示例>>Python>>正文


Python InlineMRJobRunner.stream_output方法代码示例

本文整理汇总了Python中mrjob.inline.InlineMRJobRunner.stream_output方法的典型用法代码示例。如果您正苦于以下问题:Python InlineMRJobRunner.stream_output方法的具体用法?Python InlineMRJobRunner.stream_output怎么用?Python InlineMRJobRunner.stream_output使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在mrjob.inline.InlineMRJobRunner的用法示例。


在下文中一共展示了InlineMRJobRunner.stream_output方法的5个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。

示例1: test_stream_output

# 需要导入模块: from mrjob.inline import InlineMRJobRunner [as 别名]
# 或者: from mrjob.inline.InlineMRJobRunner import stream_output [as 别名]
    def test_stream_output(self):
        a_dir_path = os.path.join(self.tmp_dir, 'a')
        b_dir_path = os.path.join(self.tmp_dir, 'b')
        l_dir_path = os.path.join(self.tmp_dir, '_logs')
        os.mkdir(a_dir_path)
        os.mkdir(b_dir_path)
        os.mkdir(l_dir_path)

        a_file_path = os.path.join(a_dir_path, 'part-00000')
        b_file_path = os.path.join(b_dir_path, 'part-00001')
        c_file_path = os.path.join(self.tmp_dir, 'part-00002')
        x_file_path = os.path.join(l_dir_path, 'log.xml')
        y_file_path = os.path.join(self.tmp_dir, '_SUCCESS')

        with open(a_file_path, 'w') as f:
            f.write('A')

        with open(b_file_path, 'w') as f:
            f.write('B')

        with open(c_file_path, 'w') as f:
            f.write('C')

        with open(x_file_path, 'w') as f:
            f.write('<XML XML XML/>')

        with open(y_file_path, 'w') as f:
            f.write('I win')

        runner = InlineMRJobRunner(conf_paths=[], output_dir=self.tmp_dir)
        self.assertEqual(sorted(runner.stream_output()),
                         [b'A', b'B', b'C'])
开发者ID:anirudhreddy92,项目名称:mrjob,代码行数:34,代码来源:test_runner.py

示例2: test_stream_output

# 需要导入模块: from mrjob.inline import InlineMRJobRunner [as 别名]
# 或者: from mrjob.inline.InlineMRJobRunner import stream_output [as 别名]
    def test_stream_output(self):
        a_dir_path = os.path.join(self.tmp_dir, "a")
        b_dir_path = os.path.join(self.tmp_dir, "b")
        l_dir_path = os.path.join(self.tmp_dir, "_logs")
        os.mkdir(a_dir_path)
        os.mkdir(b_dir_path)
        os.mkdir(l_dir_path)

        a_file_path = os.path.join(a_dir_path, "part-00000")
        b_file_path = os.path.join(b_dir_path, "part-00001")
        c_file_path = os.path.join(self.tmp_dir, "part-00002")
        x_file_path = os.path.join(l_dir_path, "log.xml")
        y_file_path = os.path.join(self.tmp_dir, "_SUCCESS")

        with open(a_file_path, "w") as f:
            f.write("A")

        with open(b_file_path, "w") as f:
            f.write("B")

        with open(c_file_path, "w") as f:
            f.write("C")

        with open(x_file_path, "w") as f:
            f.write("<XML XML XML/>")

        with open(y_file_path, "w") as f:
            f.write("I win")

        runner = InlineMRJobRunner(conf_paths=[], output_dir=self.tmp_dir)
        self.assertEqual(sorted(runner.stream_output()), ["A", "B", "C"])
开发者ID:pyzen,项目名称:mrjob,代码行数:33,代码来源:test_runner.py

示例3: test_output_dir_not_considered_hidden

# 需要导入模块: from mrjob.inline import InlineMRJobRunner [as 别名]
# 或者: from mrjob.inline.InlineMRJobRunner import stream_output [as 别名]
    def test_output_dir_not_considered_hidden(self):
        output_dir = os.path.join(self.tmp_dir, '_hidden', '_output_dir')

        self.makefile(os.path.join(output_dir, 'part-00000'),
                      b'cats\n')

        runner = InlineMRJobRunner(conf_paths=[], output_dir=output_dir)

        self.assertEqual(sorted(to_lines(runner.stream_output())),
                         [b'cats\n'])
开发者ID:Affirm,项目名称:mrjob,代码行数:12,代码来源:test_runner.py

示例4: test_deprecated_stream_output

# 需要导入模块: from mrjob.inline import InlineMRJobRunner [as 别名]
# 或者: from mrjob.inline.InlineMRJobRunner import stream_output [as 别名]
    def test_deprecated_stream_output(self):
        self.makefile('part-00000', contents=b'1\n2')
        self.makefile('part-00001', contents=b'3\n4\n')

        runner = InlineMRJobRunner(conf_paths=[], output_dir=self.tmp_dir)

        log = self.start(patch('mrjob.runner.log'))

        # should group output into lines, but not join across files
        self.assertEqual(sorted(runner.stream_output()),
                         [b'1\n', b'2', b'3\n', b'4\n'])

        # should issue deprecation warning
        self.assertEqual(log.warning.call_count, 1)
开发者ID:okomestudio,项目名称:mrjob,代码行数:16,代码来源:test_runner.py

示例5: TestCatOutput

# 需要导入模块: from mrjob.inline import InlineMRJobRunner [as 别名]
# 或者: from mrjob.inline.InlineMRJobRunner import stream_output [as 别名]
class TestCatOutput(SandboxedTestCase):

    def setUp(self):
        super(TestCatOutput, self).setUp()

        self.output_dir = os.path.join(self.tmp_dir, 'job_output')
        os.mkdir(self.output_dir)

        self.runner = InlineMRJobRunner(
            conf_paths=[], output_dir=self.output_dir)

    def test_empty(self):
        self.assertEqual(list(self.runner.cat_output()), [])

    def test_typical_output(self):
        # actual output
        self.makefile(os.path.join(self.output_dir, 'part-00000'),
                      b'line0\n')
        self.makefile(os.path.join(self.output_dir, 'part-00001'),
                      b'line1\n')

        # hidden .crc file
        self.makefile(os.path.join(self.output_dir, '.crc.part-00000'),
                      b'42\n')

        # hidden _SUCCESS file (ignore)
        self.makefile(os.path.join(self.output_dir, '_SUCCESS'),
                      b'such a relief!\n')

        # hidden _logs dir
        self.makefile(os.path.join(self.output_dir, '_logs', 'log.xml'),
                      b'pretty much the usual\n')

        self.assertEqual(sorted(to_lines(self.runner.cat_output())),
                         [b'line0\n', b'line1\n'])

    def test_output_in_subdirs(self):
        # test for output being placed in subdirs, for example with nicknack
        self.makefile(os.path.join(self.output_dir, 'a', 'part-00000'),
                      b'line-a0\n')
        self.makefile(os.path.join(self.output_dir, 'a', 'part-00001'),
                      b'line-a1\n')

        self.makefile(os.path.join(self.output_dir, 'b', 'part-00000'),
                      b'line-b0\n')

        self.makefile(os.path.join(self.output_dir, 'b', '.crc.part-00000'),
                      b'42\n')

        self.assertEqual(sorted(to_lines(self.runner.cat_output())),
                         [b'line-a0\n', b'line-a1\n', b'line-b0\n'])

    def test_read_all_non_hidden_files(self):
        self.makefile(os.path.join(self.output_dir, 'baz'),
                      b'qux\n')

        self.makefile(os.path.join(self.output_dir, 'foo', 'bar'),
                      b'baz\n')

        self.assertEqual(sorted(to_lines(self.runner.cat_output())),
                         [b'baz\n', b'qux\n'])

    def test_empty_string_between_files(self):
        self.makefile(os.path.join(self.output_dir, 'part-00000'), b'A')
        self.makefile(os.path.join(self.output_dir, 'part-00001'), b'\n')
        self.makefile(os.path.join(self.output_dir, 'part-00002'), b'C')

        # order isn't guaranteed, but there should be 3 chunks separated
        # by two empty strings
        chunks = list(self.runner.cat_output())
        self.assertEqual(len(chunks), 5)
        self.assertEqual(chunks[1], b'')
        self.assertEqual(chunks[3], b'')

    def test_output_dir_not_considered_hidden(self):
        output_dir = os.path.join(self.tmp_dir, '_hidden', '_output_dir')

        self.makefile(os.path.join(output_dir, 'part-00000'),
                      b'cats\n')

        runner = InlineMRJobRunner(conf_paths=[], output_dir=output_dir)

        self.assertEqual(sorted(to_lines(runner.stream_output())),
                         [b'cats\n'])

    def test_deprecated_stream_output(self):
        self.makefile(os.path.join(self.output_dir, 'part-00000'),
                      b'1\n2')
        self.makefile(os.path.join(self.output_dir, 'part-00001'),
                      b'3\n4\n')

        log = self.start(patch('mrjob.runner.log'))

        # should group output into lines, but not join across files
        self.assertEqual(sorted(self.runner.stream_output()),
                         [b'1\n', b'2', b'3\n', b'4\n'])

        # should issue deprecation warning
        self.assertEqual(log.warning.call_count, 1)
开发者ID:Affirm,项目名称:mrjob,代码行数:101,代码来源:test_runner.py


注:本文中的mrjob.inline.InlineMRJobRunner.stream_output方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。