Python conf.dump_mrjob_conf函数代码示例

    def test_round_trip(self):
        conf = {'runners': {'foo': {'qux': 'quux'}}}
        conf_path = os.path.join(self.tmp_dir, 'mrjob.conf')

        dump_mrjob_conf(conf, open(conf_path, 'w'))
        with no_handlers_for_logger('mrjob.conf'):
            self.assertEqual(conf, load_mrjob_conf(conf_path=conf_path))

    def test_duplicate_conf_path(self):
        conf_path = os.path.join(self.tmp_dir, "mrjob.conf")

        with open(conf_path, "w") as f:
            dump_mrjob_conf({}, f)

        self.assertEqual(load_opts_from_mrjob_confs("foo", [conf_path, conf_path]), [(conf_path, {})])

    def test_include_relative_to_real_path(self):
        os.mkdir(os.path.join(self.tmp_dir, 'conf'))

        base_conf_path = os.path.join(self.tmp_dir, 'conf', 'mrjob.base.conf')
        real_base_conf_path = os.path.realpath(base_conf_path)

        conf_path = os.path.join(self.tmp_dir, 'conf', 'mrjob.conf')
        conf_symlink_path = os.path.join(self.tmp_dir, 'mrjob.conf')

        with open(base_conf_path, 'w') as f:
            dump_mrjob_conf({}, f)

        with open(conf_path, 'w') as f:
            dump_mrjob_conf({'include': 'mrjob.base.conf'}, f)

        os.symlink(os.path.join('conf', 'mrjob.conf'), conf_symlink_path)

        self.assertEqual(
            load_opts_from_mrjob_conf('foo', conf_path),
            [(real_base_conf_path, {}), (conf_path, {})])

        # relative include should work from the symlink even though
        # it's not in the same directory as mrjob.base.conf
        self.assertEqual(
            load_opts_from_mrjob_conf('foo', conf_symlink_path),
            [(real_base_conf_path, {}), (conf_symlink_path, {})])

    def test_can_turn_off_bootstrap_mrjob(self):
        # track the dir we're loading mrjob from rather than the full path
        # to deal with edge cases where we load from the .py file,
        # and the script loads from the .pyc compiled from that .py file.
        our_mrjob_dir = os.path.dirname(os.path.realpath(mrjob.__file__))

        self.mrjob_conf_path = os.path.join(self.tmp_dir, 'mrjob.conf')
        dump_mrjob_conf({'runners': {'local': {'bootstrap_mrjob': False}}},
                        open(self.mrjob_conf_path, 'w'))

        mr_job = MRJobWhereAreYou(['-c', self.mrjob_conf_path])
        mr_job.sandbox()

        with mr_job.make_runner() as runner:
            # sanity check
            self.assertEqual(runner.get_opts()['bootstrap_mrjob'], False)
            runner.run()

            output = list(runner.stream_output())

            self.assertEqual(len(output), 1)

            # script should load mrjob from the same place our test does
            _, script_mrjob_dir = mr_job.parse_output_line(output[0])
            self.assertEqual(our_mrjob_dir, script_mrjob_dir)

 def make_mrjob_conf(self):
     _, self.mrjob_conf_path = tempfile.mkstemp(prefix='mrjob.conf.')
     dump_mrjob_conf({'runners': {'emr': {
         'check_emr_status_every': 0.01,
         's3_scratch_uri': 's3://walrus/tmp',
         's3_sync_wait_time': 0.01,
     }}}, open(self.mrjob_conf_path, 'w'))

 def make_tmp_dir_and_mrjob_conf(self):
     self.tmp_dir = tempfile.mkdtemp()
     self.mrjob_conf_path = os.path.join(self.tmp_dir, 'mrjob.conf')
     dump_mrjob_conf({'runners': {'emr': {
         'check_emr_status_every': 0.01,
         's3_sync_wait_time': 0.01,
     }}}, open(self.mrjob_conf_path, 'w'))

    def _test_round_trip(self, conf):
        conf_path = os.path.join(self.tmp_dir, "mrjob.conf")

        with open(conf_path, "w") as f:
            dump_mrjob_conf(conf, f)
        with no_handlers_for_logger("mrjob.conf"):
            self.assertEqual(conf, load_mrjob_conf(conf_path=conf_path))

    def _test_round_trip(self, conf):
        conf_path = os.path.join(self.tmp_dir, 'mrjob.conf')

        with open(conf_path, 'w') as f:
            dump_mrjob_conf(conf, f)

        self.assertEqual(conf, load_mrjob_conf(conf_path=conf_path))

    def test_recursive_include(self):
        conf_path = os.path.join(self.tmp_dir, 'mrjob.conf')
        with open(conf_path, 'w') as f:
            dump_mrjob_conf({'include': conf_path}, f)

        self.assertEqual(
            load_opts_from_mrjob_conf('foo', conf_path),
            [(conf_path, {})])

    def test_using_json_and_not_yaml(self):
        conf = {"runners": {"foo": {"qux": "quux"}}}
        conf_path = os.path.join(self.tmp_dir, "mrjob.conf")

        dump_mrjob_conf(conf, open(conf_path, "w"))
        with open(conf_path) as f:
            contents = f.read()

        self.assertEqual(contents.replace(" ", "").replace("\n", ""), '{"runners":{"foo":{"qux":"quux"}}}')

 def make_tmp_dir_and_mrjob_conf(self):
     self.tmp_dir = tempfile.mkdtemp()
     self.mrjob_conf_path = os.path.join(self.tmp_dir, 'mrjob.conf')
     dump_mrjob_conf({'runners': {'emr': {
         'check_emr_status_every': 0.01,
         's3_sync_wait_time': 0.01,
         'aws_availability_zone': 'PUPPYLAND',
         'additional_emr_info': {'key': 'value'},
     }}}, open(self.mrjob_conf_path, 'w'))

    def test_using_json_and_not_yaml(self):
        conf = {'runners': {'foo': {'qux': 'quux'}}}
        conf_path = os.path.join(self.tmp_dir, 'mrjob.conf')

        dump_mrjob_conf(conf, open(conf_path, 'w'))
        contents = open(conf_path).read()

        assert_equal(contents.replace(' ', '').replace('\n', ''),
                     '{"runners":{"foo":{"qux":"quux"}}}')

    def test_doubly_recursive_include(self):
        conf_path_1 = os.path.join(self.tmp_dir, "mrjob.1.conf")
        conf_path_2 = os.path.join(self.tmp_dir, "mrjob.2.conf")

        with open(conf_path_1, "w") as f:
            dump_mrjob_conf({"include": conf_path_2}, f)

        with open(conf_path_2, "w") as f:
            dump_mrjob_conf({"include": conf_path_1}, f)

        self.assertEqual(load_opts_from_mrjob_conf("foo", conf_path_1), [(conf_path_2, {}), (conf_path_1, {})])

    def test_recurse(self):
        path = os.path.join(self.tmp_dir, 'LOL.conf')
        recurse_conf = dict(include=path)
        with open(path, 'w') as f:
            dump_mrjob_conf(recurse_conf, f)

        stderr = StringIO()
        with no_handlers_for_logger():
            log_to_stream('mrjob.conf', stderr)
            InlineMRJobRunner(conf_path=path)
            self.assertIn('%s tries to recursively include %s!' % (path, path),
                          stderr.getvalue())

    def test_relative_include(self):
        base_conf_path = os.path.join(self.tmp_dir, "mrjob.base.conf")
        real_base_conf_path = os.path.realpath(base_conf_path)

        conf_path = os.path.join(self.tmp_dir, "mrjob.conf")

        with open(base_conf_path, "w") as f:
            dump_mrjob_conf({}, f)

        with open(conf_path, "w") as f:
            dump_mrjob_conf({"include": "mrjob.base.conf"}, f)

        self.assertEqual(load_opts_from_mrjob_conf("foo", conf_path), [(real_base_conf_path, {}), (conf_path, {})])