Python StringIO.getvalue方法代码示例

# 需要导入模块: from mrjob.py2 import StringIO [as 别名]
# 或者: from mrjob.py2.StringIO import getvalue [as 别名]
    def test_cleanup_options(self):
        stderr = StringIO()
        with no_handlers_for_logger('mrjob.runner'):
            log_to_stream('mrjob.runner', stderr)
            opts = RunnerOptionStore(
                'inline',
                dict(cleanup=['LOCAL_SCRATCH', 'REMOTE_SCRATCH'],
                     cleanup_on_failure=['JOB_FLOW', 'SCRATCH']),
                [])

            self.assertEqual(opts['cleanup'], ['LOCAL_TMP', 'CLOUD_TMP'])
            self.assertIn(
                'Deprecated cleanup option LOCAL_SCRATCH has been renamed'
                ' to LOCAL_TMP', stderr.getvalue())
            self.assertIn(
                'Deprecated cleanup option REMOTE_SCRATCH has been renamed'
                ' to CLOUD_TMP', stderr.getvalue())

            self.assertEqual(opts['cleanup_on_failure'], ['CLUSTER', 'TMP'])
            self.assertIn(
                'Deprecated cleanup_on_failure option JOB_FLOW has been'
                ' renamed to CLUSTER', stderr.getvalue())
            self.assertIn(
                'Deprecated cleanup_on_failure option SCRATCH has been renamed'
                ' to TMP', stderr.getvalue())

# 需要导入模块: from mrjob.py2 import StringIO [as 别名]
# 或者: from mrjob.py2.StringIO import getvalue [as 别名]
    def test_option_debug_printout(self):
        stderr = StringIO()

        with no_handlers_for_logger():
            log_to_stream('mrjob.runner', stderr, debug=True)

            InlineMRJobRunner(owner='dave')

        self.assertIn("'owner'", stderr.getvalue())
        self.assertIn("'dave'", stderr.getvalue())

# 需要导入模块: from mrjob.py2 import StringIO [as 别名]
# 或者: from mrjob.py2.StringIO import getvalue [as 别名]
    def test_prefer_own_methods(self):
        # TODO: currently can't initialize HadoopRunner without setting these
        runner = HadoopJobRunner(hadoop_bin="hadoop", hadoop_home="kansas", hadoop_streaming_jar="streaming.jar")

        with no_handlers_for_logger("mrjob.runner"):
            stderr = StringIO()
            log_to_stream("mrjob.runner", stderr)

            self.assertEqual(runner.ls, runner.fs.ls)

            # Hadoop Runner has its own version
            self.assertNotEqual(runner.get_hadoop_version, runner.fs.get_hadoop_version)

            self.assertIn("deprecated: call HadoopJobRunner.fs.ls() directly", stderr.getvalue())
            self.assertNotIn("get_hadoop_version", stderr.getvalue())

# 需要导入模块: from mrjob.py2 import StringIO [as 别名]
# 或者: from mrjob.py2.StringIO import getvalue [as 别名]
class ReportLongJobsTestCase(MockBotoTestCase):

    def setUp(self):
        super(ReportLongJobsTestCase, self).setUp()
        # redirect print statements to self.stdout
        self._real_stdout = sys.stdout
        self.stdout = StringIO()
        sys.stdout = self.stdout

    def tearDown(self):
        sys.stdout = self._real_stdout
        super(ReportLongJobsTestCase, self).tearDown()

    def test_with_no_clusters(self):
        main(['-q', '--no-conf'])  # just make sure it doesn't crash

    def test_with_all_clusters(self):
        for cluster in CLUSTERS:
            self.add_mock_emr_cluster(cluster)

        emr_conn = self.connect_emr()
        emr_conn.run_jobflow('no name',
                             job_flow_role='fake-instance-profile',
                             service_role='fake-service-role')
        main(['-q', '--no-conf'])

        lines = [line for line in StringIO(self.stdout.getvalue())]
        self.assertEqual(len(lines), len(CLUSTERS_BY_ID) - 1)

# 需要导入模块: from mrjob.py2 import StringIO [as 别名]
# 或者: from mrjob.py2.StringIO import getvalue [as 别名]
    def test_non_log_lines(self):
        lines = StringIO('foo\n'
                         'bar\n'
                         '15/12/11 13:26:08 ERROR streaming.StreamJob:'
                         ' Error Launching job :'
                         ' Output directory already exists\n'
                         'Streaming Command Failed!')

        with no_handlers_for_logger('mrjob.logs.parse'):
            stderr = StringIO()
            log_to_stream('mrjob.logs.parse', stderr)

            self.assertEqual(
            list(_parse_hadoop_log_lines(lines)), [
                # ignore leading non-log lines
                dict(
                    timestamp='15/12/11 13:26:08',
                    level='ERROR',
                    logger='streaming.StreamJob',
                    thread=None,
                    # no way to know that Streaming Command Failed! wasn't part
                    # of a multi-line message
                    message=('Error Launching job :'
                             ' Output directory already exists\n'
                             'Streaming Command Failed!'))
            ])

            # should be one warning for each leading non-log line
            log_lines = stderr.getvalue().splitlines()
            self.assertEqual(len(log_lines), 2)

# 需要导入模块: from mrjob.py2 import StringIO [as 别名]
# 或者: from mrjob.py2.StringIO import getvalue [as 别名]
    def test_non_log_lines(self):
        lines = StringIO(
            "foo\n"
            "bar\n"
            "15/12/11 13:26:08 ERROR streaming.StreamJob:"
            " Error Launching job :"
            " Output directory already exists\n"
            "Streaming Command Failed!"
        )

        with no_handlers_for_logger("mrjob.logs.parse"):
            stderr = StringIO()
            log_to_stream("mrjob.logs.parse", stderr)

            self.assertEqual(
                list(_parse_hadoop_log_lines(lines)),
                [
                    # ignore leading non-log lines
                    dict(
                        timestamp="15/12/11 13:26:08",
                        level="ERROR",
                        logger="streaming.StreamJob",
                        thread=None,
                        # no way to know that Streaming Command Failed! wasn't part
                        # of a multi-line message
                        message=(
                            "Error Launching job :" " Output directory already exists\n" "Streaming Command Failed!"
                        ),
                    )
                ],
            )

            # should be one warning for each leading non-log line
            log_lines = stderr.getvalue().splitlines()
            self.assertEqual(len(log_lines), 2)

# 需要导入模块: from mrjob.py2 import StringIO [as 别名]
# 或者: from mrjob.py2.StringIO import getvalue [as 别名]
    def test_failed_job(self):
        mr_job = MRTwoStepJob(['-r', 'dataproc', '-v'])
        mr_job.sandbox()

        with no_handlers_for_logger('mrjob.dataproc'):
            stderr = StringIO()
            log_to_stream('mrjob.dataproc', stderr)

            self._dataproc_client.job_get_advances_states = (
                collections.deque(['SETUP_DONE', 'RUNNING', 'ERROR']))

            with mr_job.make_runner() as runner:
                self.assertIsInstance(runner, DataprocJobRunner)

                self.assertRaises(StepFailedException, runner.run)

                self.assertIn(' => ERROR\n', stderr.getvalue())

                cluster_id = runner.get_cluster_id()

        # job should get terminated
        cluster = (
            self._dataproc_client._cache_clusters[_TEST_PROJECT][cluster_id])
        cluster_state = self._dataproc_client.get_state(cluster)
        self.assertEqual(cluster_state, 'DELETING')

# 需要导入模块: from mrjob.py2 import StringIO [as 别名]
# 或者: from mrjob.py2.StringIO import getvalue [as 别名]
 def assert_hadoop_version(self, JobClass, version_string):
     mr_job = JobClass()
     mock_log = StringIO()
     with no_handlers_for_logger("mrjob.job"):
         log_to_stream("mrjob.job", mock_log)
         self.assertEqual(mr_job.jobconf()["hadoop_version"], version_string)
         self.assertIn("should be a string", mock_log.getvalue())

# 需要导入模块: from mrjob.py2 import StringIO [as 别名]
# 或者: from mrjob.py2.StringIO import getvalue [as 别名]
class ReportLongJobsTestCase(MockBoto3TestCase):

    def setUp(self):
        super(ReportLongJobsTestCase, self).setUp()
        # redirect print statements to self.stdout
        self._real_stdout = sys.stdout
        self.stdout = StringIO()
        sys.stdout = self.stdout

    def tearDown(self):
        sys.stdout = self._real_stdout
        super(ReportLongJobsTestCase, self).tearDown()

    def test_with_no_clusters(self):
        main(['-q', '--no-conf'])  # just make sure it doesn't crash

    def test_with_all_clusters(self):
        for cluster in CLUSTERS:
            self.add_mock_emr_cluster(cluster)

        emr_client = self.client('emr')
        emr_client.run_job_flow(
            Name='no name',
            Instances=dict(
                MasterInstanceType='m1.medium',
                InstanceCount=1,
            ),
            JobFlowRole='fake-instance-profile',
            ReleaseLabel='emr-4.0.0',
            ServiceRole='fake-service-role',
        )
        main(['-q', '--no-conf'])

        lines = [line for line in StringIO(self.stdout.getvalue())]
        self.assertEqual(len(lines), len(CLUSTERS_BY_ID) - 1)
        self.assertNotIn('j-COMPLETED', self.stdout.getvalue())

    def test_exclude(self):
        for cluster in CLUSTERS:
            self.add_mock_emr_cluster(cluster)

        main(['-q', '--no-conf', '-x', 'my_key,my_value'])

        lines = [line for line in StringIO(self.stdout.getvalue())]
        self.assertEqual(len(lines), len(CLUSTERS_BY_ID) - 2)
        self.assertNotIn('j-COMPLETED', self.stdout.getvalue())
        self.assertNotIn('j-RUNNING1STEP', self.stdout.getvalue())

# 需要导入模块: from mrjob.py2 import StringIO [as 别名]
# 或者: from mrjob.py2.StringIO import getvalue [as 别名]
    def test_passthrough(self):
        runner = InlineMRJobRunner()

        with no_handlers_for_logger('mrjob.runner'):
            stderr = StringIO()
            log_to_stream('mrjob.runner', stderr)

            self.assertEqual(runner.ls, runner.fs.ls)
            # no special rules for underscore methods
            self.assertEqual(runner._cat_file, runner.fs._cat_file)

            self.assertIn(
                'deprecated: call InlineMRJobRunner.fs.ls() directly',
                stderr.getvalue())
            self.assertIn(
                'deprecated: call InlineMRJobRunner.fs._cat_file() directly',
                stderr.getvalue())

# 需要导入模块: from mrjob.py2 import StringIO [as 别名]
# 或者: from mrjob.py2.StringIO import getvalue [as 别名]
    def updated_and_warnings(self, jobconf, hadoop_version):
        jobconf = jobconf.copy()
        with no_handlers_for_logger("mrjob.runner"):
            stderr = StringIO()
            log_to_stream("mrjob.runner", stderr)
            self.runner._update_jobconf_for_hadoop_version(jobconf, hadoop_version)

        return jobconf, stderr.getvalue()

# 需要导入模块: from mrjob.py2 import StringIO [as 别名]
# 或者: from mrjob.py2.StringIO import getvalue [as 别名]
 def test_messy_error(self):
     counter_string = b'Job JOBID="_001" FAILED_REDUCES="0" COUNTERS="THIS IS NOT ACTUALLY A COUNTER"'
     with no_handlers_for_logger(''):
         stderr = StringIO()
         log_to_stream('mrjob.parse', stderr, level=logging.WARN)
         self.assertEqual(({}, 1),
                          parse_hadoop_counters_from_line(counter_string))
         self.assertIn('Cannot parse Hadoop counter string',
                       stderr.getvalue())

# 需要导入模块: from mrjob.py2 import StringIO [as 别名]
# 或者: from mrjob.py2.StringIO import getvalue [as 别名]
    def test_hadoop_runner_option_store(self):
        stderr = StringIO()
        with no_handlers_for_logger('mrjob.conf'):
            log_to_stream('mrjob.conf', stderr)

            opts = HadoopRunnerOptionStore(
                    'hadoop', dict(base_tmp_dir='/scratch',
                         hdfs_scratch_dir='hdfs:///scratch',
                    ), [])

            self.assertEqual(opts['local_tmp_dir'], '/scratch')
            self.assertNotIn('base_tmp_dir', opts)
            self.assertIn('Deprecated option base_tmp_dir has been renamed'
                          ' to local_tmp_dir', stderr.getvalue())

            self.assertEqual(opts['hadoop_tmp_dir'], 'hdfs:///scratch')
            self.assertNotIn('hdfs_scratch_dir', opts)
            self.assertIn('Deprecated option hdfs_scratch_dir has been renamed'
                          ' to hadoop_tmp_dir', stderr.getvalue())

# 需要导入模块: from mrjob.py2 import StringIO [as 别名]
# 或者: from mrjob.py2.StringIO import getvalue [as 别名]
    def get_debug_printout(self, opt_store_class, alias, opts):
        stderr = StringIO()

        with no_handlers_for_logger():
            log_to_stream('mrjob.runner', stderr, debug=True)

            # debug printout happens in constructor
            opt_store_class(alias, opts, [])

        return stderr.getvalue()

# 需要导入模块: from mrjob.py2 import StringIO [as 别名]
# 或者: from mrjob.py2.StringIO import getvalue [as 别名]
    def test_emr_runner_option_store(self):
        stderr = StringIO()
        with no_handlers_for_logger('mrjob.conf'):
            log_to_stream('mrjob.conf', stderr)

            opts = EMRRunnerOptionStore(
                'emr',
                dict(base_tmp_dir='/scratch',
                     s3_scratch_uri='s3://bucket/walrus'),
                    [])

            self.assertEqual(opts['local_tmp_dir'], '/scratch')
            self.assertNotIn('base_tmp_dir', opts)
            self.assertIn('Deprecated option base_tmp_dir has been renamed'
                          ' to local_tmp_dir', stderr.getvalue())

            self.assertEqual(opts['s3_tmp_dir'], 's3://bucket/walrus')
            self.assertNotIn('s3_scratch_uri', opts)
            self.assertIn('Deprecated option s3_scratch_uri has been renamed'
                          ' to s3_tmp_dir', stderr.getvalue())