Python SystemResourceMonitor.as_dict方法代码示例

# 需要导入模块: from mozsystemmonitor.resourcemonitor import SystemResourceMonitor [as 别名]
# 或者: from mozsystemmonitor.resourcemonitor.SystemResourceMonitor import as_dict [as 别名]
    def test_as_dict(self):
        monitor = SystemResourceMonitor(poll_interval=0.25)

        monitor.start()
        time.sleep(0.1)
        monitor.begin_phase('phase1')
        monitor.record_event('foo')
        time.sleep(0.1)
        monitor.begin_phase('phase2')
        monitor.record_event('bar')
        time.sleep(0.2)
        monitor.finish_phase('phase1')
        time.sleep(0.2)
        monitor.finish_phase('phase2')
        time.sleep(0.4)
        monitor.stop()

        d = monitor.as_dict()

        self.assertEqual(d['version'], 2)
        self.assertEqual(len(d['events']), 2)
        self.assertEqual(len(d['phases']), 2)
        self.assertIn('system', d)
        self.assertIsInstance(d['system'], dict)
        self.assertIsInstance(d['overall'], dict)
        self.assertIn('duration', d['overall'])
        self.assertIn('cpu_times', d['overall'])

# 需要导入模块: from mozsystemmonitor.resourcemonitor import SystemResourceMonitor [as 别名]
# 或者: from mozsystemmonitor.resourcemonitor.SystemResourceMonitor import as_dict [as 别名]
    def test_as_dict(self):
        monitor = SystemResourceMonitor(poll_interval=0.25)

        monitor.start()
        time.sleep(0.1)
        monitor.begin_phase('phase1')
        monitor.record_event('foo')
        time.sleep(0.1)
        monitor.begin_phase('phase2')
        monitor.record_event('bar')
        time.sleep(0.2)
        monitor.finish_phase('phase1')
        time.sleep(0.2)
        monitor.finish_phase('phase2')
        time.sleep(0.4)
        monitor.stop()

        d = monitor.as_dict()

        self.assertEqual(d['version'], 1)
        self.assertEqual(len(d['events']), 2)
        self.assertEqual(len(d['phases']), 2)

# 需要导入模块: from mozsystemmonitor.resourcemonitor import SystemResourceMonitor [as 别名]
# 或者: from mozsystemmonitor.resourcemonitor.SystemResourceMonitor import as_dict [as 别名]
class ResourceMonitoringMixin(PerfherderResourceOptionsMixin):
    """Provides resource monitoring capabilities to scripts.

    When this class is in the inheritance chain, resource usage stats of the
    executing script will be recorded.

    This class requires the VirtualenvMixin in order to install a package used
    for recording resource usage.

    While we would like to record resource usage for the entirety of a script,
    since we require an external package, we can only record resource usage
    after that package is installed (as part of creating the virtualenv).
    That's just the way things have to be.
    """
    def __init__(self, *args, **kwargs):
        super(ResourceMonitoringMixin, self).__init__(*args, **kwargs)

        self.register_virtualenv_module('psutil>=3.1.1', method='pip',
                                        optional=True)
        self.register_virtualenv_module('mozsystemmonitor==0.3',
                                        method='pip', optional=True)
        self.register_virtualenv_module('jsonschema==2.5.1',
                                        method='pip')
        # explicitly install functools32, because some slaves aren't using
        # a version of pip recent enough to install it automatically with
        # jsonschema (which depends on it)
        # https://github.com/Julian/jsonschema/issues/233
        self.register_virtualenv_module('functools32==3.2.3-2',
                                        method='pip')
        self._resource_monitor = None

        # 2-tuple of (name, options) to assign Perfherder resource monitor
        # metrics to. This needs to be assigned by a script in order for
        # Perfherder metrics to be reported.
        self.resource_monitor_perfherder_id = None

    @PostScriptAction('create-virtualenv')
    def _start_resource_monitoring(self, action, success=None):
        self.activate_virtualenv()

        # Resource Monitor requires Python 2.7, however it's currently optional.
        # Remove when all machines have had their Python version updated (bug 711299).
        if sys.version_info[:2] < (2, 7):
            self.warning('Resource monitoring will not be enabled! Python 2.7+ required.')
            return

        try:
            from mozsystemmonitor.resourcemonitor import SystemResourceMonitor

            self.info("Starting resource monitoring.")
            self._resource_monitor = SystemResourceMonitor(poll_interval=1.0)
            self._resource_monitor.start()
        except Exception:
            self.warning("Unable to start resource monitor: %s" %
                         traceback.format_exc())

    @PreScriptAction
    def _resource_record_pre_action(self, action):
        # Resource monitor isn't available until after create-virtualenv.
        if not self._resource_monitor:
            return

        self._resource_monitor.begin_phase(action)

    @PostScriptAction
    def _resource_record_post_action(self, action, success=None):
        # Resource monitor isn't available until after create-virtualenv.
        if not self._resource_monitor:
            return

        self._resource_monitor.finish_phase(action)

    @PostScriptRun
    def _resource_record_post_run(self):
        if not self._resource_monitor:
            return

        # This should never raise an exception. This is a workaround until
        # mozsystemmonitor is fixed. See bug 895388.
        try:
            self._resource_monitor.stop()
            self._log_resource_usage()

            # Upload a JSON file containing the raw resource data.
            try:
                upload_dir = self.query_abs_dirs()['abs_blob_upload_dir']
                if not os.path.exists(upload_dir):
                    os.makedirs(upload_dir)
                with open(os.path.join(upload_dir, 'resource-usage.json'), 'wb') as fh:
                    json.dump(self._resource_monitor.as_dict(), fh,
                              sort_keys=True, indent=4)
            except (AttributeError, KeyError):
                self.exception('could not upload resource usage JSON',
                               level=WARNING)

        except Exception:
            self.warning("Exception when reporting resource usage: %s" %
                         traceback.format_exc())

    def _log_resource_usage(self):
#.........这里部分代码省略.........

# 需要导入模块: from mozsystemmonitor.resourcemonitor import SystemResourceMonitor [as 别名]
# 或者: from mozsystemmonitor.resourcemonitor.SystemResourceMonitor import as_dict [as 别名]

#.........这里部分代码省略.........
        try:
            warning = self._warnings_collector.process_line(line)
        except:
            pass

        return BuildOutputResult(warning, False, True)

    def stop_resource_recording(self):
        if self._resources_started:
            self.resources.stop()

        self._resources_started = False

    def finish(self, record_usage=True):
        """Record the end of the build."""
        self.stop_resource_recording()
        self.end_time = time.time()
        self._finder_end_cpu = self._get_finder_cpu_usage()
        self.elapsed = self.end_time - self.start_time

        self.warnings_database.prune()
        self.warnings_database.save_to_file(self._warnings_path)

        if not record_usage:
            return

        try:
            usage = self.get_resource_usage()
            if not usage:
                return

            self.log_resource_usage(usage)
            with open(self._get_state_filename('build_resources.json'), 'w') as fh:
                json.dump(self.resources.as_dict(), fh, indent=2)
        except Exception as e:
            self.log(logging.WARNING, 'build_resources_error',
                {'msg': str(e)},
                'Exception when writing resource usage file: {msg}')

    def _get_finder_cpu_usage(self):
        """Obtain the CPU usage of the Finder app on OS X.

        This is used to detect high CPU usage.
        """
        if not sys.platform.startswith('darwin'):
            return None

        if not psutil:
            return None

        for proc in psutil.process_iter():
            if proc.name != 'Finder':
                continue

            if proc.username != getpass.getuser():
                continue

            # Try to isolate system finder as opposed to other "Finder"
            # processes.
            if not proc.exe.endswith('CoreServices/Finder.app/Contents/MacOS/Finder'):
                continue

            return proc.get_cpu_times()

        return None

# 需要导入模块: from mozsystemmonitor.resourcemonitor import SystemResourceMonitor [as 别名]
# 或者: from mozsystemmonitor.resourcemonitor.SystemResourceMonitor import as_dict [as 别名]
class ResourceMonitoringMixin(object):
    """Provides resource monitoring capabilities to scripts.

    When this class is in the inheritance chain, resource usage stats of the
    executing script will be recorded.

    This class requires the VirtualenvMixin in order to install a package used
    for recording resource usage.

    While we would like to record resource usage for the entirety of a script,
    since we require an external package, we can only record resource usage
    after that package is installed (as part of creating the virtualenv).
    That's just the way things have to be.
    """
    def __init__(self, *args, **kwargs):
        super(ResourceMonitoringMixin, self).__init__(*args, **kwargs)

        self.register_virtualenv_module('psutil>=3.1.1', method='pip',
                                        optional=True)
        self.register_virtualenv_module('mozsystemmonitor==0.3',
                                        method='pip', optional=True)
        self._resource_monitor = None

    @PostScriptAction('create-virtualenv')
    def _start_resource_monitoring(self, action, success=None):
        self.activate_virtualenv()

        # Resource Monitor requires Python 2.7, however it's currently optional.
        # Remove when all machines have had their Python version updated (bug 711299).
        if sys.version_info[:2] < (2, 7):
            self.warning('Resource monitoring will not be enabled! Python 2.7+ required.')
            return

        try:
            from mozsystemmonitor.resourcemonitor import SystemResourceMonitor

            self.info("Starting resource monitoring.")
            self._resource_monitor = SystemResourceMonitor(poll_interval=1.0)
            self._resource_monitor.start()
        except Exception:
            self.warning("Unable to start resource monitor: %s" %
                         traceback.format_exc())

    @PreScriptAction
    def _resource_record_pre_action(self, action):
        # Resource monitor isn't available until after create-virtualenv.
        if not self._resource_monitor:
            return

        self._resource_monitor.begin_phase(action)

    @PostScriptAction
    def _resource_record_post_action(self, action, success=None):
        # Resource monitor isn't available until after create-virtualenv.
        if not self._resource_monitor:
            return

        self._resource_monitor.finish_phase(action)

    @PostScriptRun
    def _resource_record_post_run(self):
        if not self._resource_monitor:
            return

        # This should never raise an exception. This is a workaround until
        # mozsystemmonitor is fixed. See bug 895388.
        try:
            self._resource_monitor.stop()
            self._log_resource_usage()

            # Upload a JSON file containing the raw resource data.
            try:
                upload_dir = self.query_abs_dirs()['abs_blob_upload_dir']
                with open(os.path.join(upload_dir, 'resource-usage.json'), 'wb') as fh:
                    json.dump(self._resource_monitor.as_dict(), fh,
                              sort_keys=True, indent=4)
            except (AttributeError, KeyError):
                self.exception('could not upload resource usage JSON',
                               level=WARNING)

        except Exception:
            self.warning("Exception when reporting resource usage: %s" %
                         traceback.format_exc())

    def _log_resource_usage(self):
        rm = self._resource_monitor

        if rm.start_time is None:
            return

        def resources(phase):
            cpu_percent = rm.aggregate_cpu_percent(phase=phase, per_cpu=False)
            cpu_times = rm.aggregate_cpu_times(phase=phase, per_cpu=False)
            io = rm.aggregate_io(phase=phase)

            swap_in = sum(m.swap.sin for m in rm.measurements)
            swap_out = sum(m.swap.sout for m in rm.measurements)

            return cpu_percent, cpu_times, io, (swap_in, swap_out)

#.........这里部分代码省略.........