Python host_select.HostSelector類代碼示例

 def __init__(self, *args, **kwargs):
     Runner.__init__(self, *args, **kwargs)
     self.host_selector = HostSelector(self.event_handler, self.popen)
     self.suite_run_cleaner = SuiteRunCleaner(
             event_handler=self.event_handler,
             host_selector=self.host_selector,
             suite_engine_proc=self.suite_engine_proc)

 def __init__(self, event_handler=None):
     self.event_handler = event_handler
     self.popen = RosePopener(self.event_handler)
     self.fs_util = FileSystemUtil(self.event_handler)
     self.config_pm = ConfigProcessorsManager(
         self.event_handler, self.popen, self.fs_util)
     self.host_selector = HostSelector(self.event_handler, self.popen)
     self.suite_engine_proc = SuiteEngineProcessor.get_processor(
         event_handler=self.event_handler,
         popen=self.popen,
         fs_util=self.fs_util)

 def __init__(self, *args, **kwargs):
     self.exposed = True
     self.suite_engine_proc = SuiteEngineProcessor.get_processor()
     rose_conf = ResourceLocator.default().get_conf()
     self.logo = rose_conf.get_value(["rose-bush", "logo"])
     self.title = rose_conf.get_value(["rose-bush", "title"], self.TITLE)
     self.host_name = rose_conf.get_value(["rose-bush", "host"])
     if self.host_name is None:
         self.host_name = HostSelector().get_local_host()
         if self.host_name and "." in self.host_name:
             self.host_name = self.host_name.split(".", 1)[0]
     self.rose_version = ResourceLocator.default().get_version()
     template_env = jinja2.Environment(loader=jinja2.FileSystemLoader(
         ResourceLocator.default().get_util_home(
             "lib", "html", "template", "rose-bush")))
     self.template_env = template_env

 def __init__(self, opts, reporter=None, popen=None, fs_util=None):
     self.opts = opts
     if reporter is None:
         self.reporter = Reporter(opts.verbosity - opts.quietness)
     else:
         self.reporter = reporter
     if popen is None:
         self.popen = RosePopener(event_handler=self.reporter)
     else:
         self.popen = popen
     if fs_util is None:
         self.fs_util = FileSystemUtil(event_handler=self.reporter)
     else:
         self.fs_util = fs_util
     self.host_selector = HostSelector(event_handler=self.reporter,
                                       popen=self.popen)

    def __init__(self, *args, **kwargs):
        self.exposed = True
        self.suite_engine_proc = SuiteEngineProcessor.get_processor()
        rose_conf = ResourceLocator.default().get_conf()
        self.logo = rose_conf.get_value(["rose-bush", "logo"])
        self.title = rose_conf.get_value(["rose-bush", "title"], self.TITLE)
        self.host_name = rose_conf.get_value(["rose-bush", "host"])
        if self.host_name is None:
            self.host_name = HostSelector().get_local_host()
            if self.host_name and "." in self.host_name:
                self.host_name = self.host_name.split(".", 1)[0]
        self.rose_version = ResourceLocator.default().get_version()
        template_env = jinja2.Environment(loader=jinja2.FileSystemLoader(
            ResourceLocator.default().get_util_home(
                "lib", "html", "template", "rose-bush")))

        def urlise(text):
            pattern = '((https?):\/\/[^\s\(\)&\[\]\{\}]+)'
            replacement = '<a href="\g<1>">\g<1></a>'
            text = re.sub(pattern, replacement, text)
            return text

        template_env.filters['urlise'] = urlise
        self.template_env = template_env

class SuiteRestarter(object):

    """Wrap "cylc restart"."""

    def __init__(self, event_handler=None):
        self.event_handler = event_handler
        self.popen = RosePopener(self.event_handler)
        self.fs_util = FileSystemUtil(self.event_handler)
        self.config_pm = ConfigProcessorsManager(
            self.event_handler, self.popen, self.fs_util)
        self.host_selector = HostSelector(self.event_handler, self.popen)
        self.suite_engine_proc = SuiteEngineProcessor.get_processor(
            event_handler=self.event_handler,
            popen=self.popen,
            fs_util=self.fs_util)

    def handle_event(self, *args, **kwargs):
        """Handle event."""
        if callable(self.event_handler):
            self.event_handler(*args, **kwargs)

    def restart(
            self, suite_name=None, host=None, gcontrol_mode=None, args=None):
        """Restart a "cylc" suite."""
        # Check suite engine specific compatibility
        self.suite_engine_proc.check_global_conf_compat()

        if not suite_name:
            suite_name = get_suite_name(self.event_handler)

        suite_dir = self.suite_engine_proc.get_suite_dir(suite_name)
        if not os.path.exists(suite_dir):
            raise SuiteNotFoundError(suite_dir)

        # Ensure suite is not running
        hosts = []
        if host:
            hosts.append(host)
        self.suite_engine_proc.check_suite_not_running(suite_name, hosts)

        # Determine suite host to restart suite
        if host:
            hosts = [host]
        else:
            hosts = []
            val = ResourceLocator.default().get_conf().get_value(
                ["rose-suite-run", "hosts"], "localhost")
            known_hosts = self.host_selector.expand(val.split())[0]
            for known_host in known_hosts:
                if known_host not in hosts:
                    hosts.append(known_host)

        if hosts == ["localhost"]:
            host = hosts[0]
        else:
            host = self.host_selector(hosts)[0][0]
        self.handle_event(SuiteHostSelectEvent(suite_name, "restart", host))

        # Suite host environment
        run_conf_file_name = self.suite_engine_proc.get_suite_dir(
            suite_name, "log", "rose-suite-run.conf")
        try:
            run_conf = ConfigLoader().load(run_conf_file_name)
        except (ConfigSyntaxError, IOError):
            environ = None
        else:
            run_conf_tree = ConfigTree()
            run_conf_tree.node = run_conf
            environ = self.config_pm(run_conf_tree, "env")

        # Restart the suite
        self.suite_engine_proc.run(suite_name, host, environ, "restart", args)

        # Write suite host name to host file
        host_file_name = self.suite_engine_proc.get_suite_dir(
            suite_name, "log", "rose-suite-run.host")
        open(host_file_name, "w").write(host + "\n")

        # Launch the monitoring tool
        # Note: maybe use os.ttyname(sys.stdout.fileno())?
        if os.getenv("DISPLAY") and host and gcontrol_mode:
            self.suite_engine_proc.gcontrol(suite_name, host)

        return

class SuiteRunner(Runner):

    """Invoke a Rose suite."""

    SLEEP_PIPE = 0.05
    NAME = "suite"
    OPTIONS = ["conf_dir", "defines", "defines_suite", "gcontrol_mode", "host",
               "install_only_mode", "local_install_only_mode",
               "log_archive_mode", "log_keep", "log_name", "name", "new_mode",
               "no_overwrite_mode", "opt_conf_keys", "reload_mode", "remote",
               "restart_mode", "run_mode", "strict_mode"]

    REC_DONT_SYNC = re.compile(
            r"\A(?:\..*|cylc-suite\.db.*|log(?:\..*)*|state|share|work)\Z")

    def __init__(self, *args, **kwargs):
        Runner.__init__(self, *args, **kwargs)
        self.host_selector = HostSelector(self.event_handler, self.popen)
        self.suite_run_cleaner = SuiteRunCleaner(
                event_handler=self.event_handler,
                host_selector=self.host_selector,
                suite_engine_proc=self.suite_engine_proc)

    def run_impl(self, opts, args, uuid, work_files):
        # Log file, temporary
        if hasattr(self.event_handler, "contexts"):
            t_file = TemporaryFile()
            log_context = ReporterContext(None, self.event_handler.VV, t_file)
            self.event_handler.contexts[uuid] = log_context

        # Check suite engine specific compatibility
        self.suite_engine_proc.check_global_conf_compat()

        # Suite name from the current working directory
        if opts.conf_dir:
            self.fs_util.chdir(opts.conf_dir)
        opts.conf_dir = os.getcwd()

        if opts.defines_suite:
            suite_section = "jinja2:" + self.suite_engine_proc.SUITE_CONF
            if not opts.defines:
                opts.defines = []
            for define in opts.defines_suite:
                opts.defines.append("[" + suite_section + "]" + define)

        # --remote=KEY=VALUE,...
        if opts.remote:
            # opts.name always set for remote.
            return self._run_remote(opts, opts.name)

        conf_tree = self.config_load(opts)
        self.fs_util.chdir(conf_tree.conf_dirs[0])

        suite_name = opts.name
        if not opts.name:
            suite_name = os.path.basename(os.getcwd())

        # Automatic Rose constants
        # ROSE_ORIG_HOST: originating host
        # ROSE_VERSION: Rose version (not retained in run_mode=="reload")
        # Suite engine version
        jinja2_section = "jinja2:" + self.suite_engine_proc.SUITE_CONF
        my_rose_version = ResourceLocator.default().get_version()
        suite_engine_key = self.suite_engine_proc.get_version_env_name()
        if opts.run_mode == "reload":
            prev_config_path = self.suite_engine_proc.get_suite_dir(
                    suite_name, "log", "rose-suite-run.conf")
            prev_config = ConfigLoader()(prev_config_path)
            suite_engine_version = prev_config.get_value(
                    ["env", suite_engine_key])
        else:
            suite_engine_version = self.suite_engine_proc.get_version()
        auto_items = {"ROSE_ORIG_HOST": socket.gethostname(),
                      "ROSE_VERSION": ResourceLocator.default().get_version(),
                      suite_engine_key: suite_engine_version}
        for key, val in auto_items.items():
            requested_value = conf_tree.node.get_value(["env", key])
            if requested_value:
                if key == "ROSE_VERSION" and val != requested_value:
                    exc = VersionMismatchError(requested_value, val)
                    raise ConfigValueError(["env", key], requested_value, exc)
                val = requested_value
            else:
                conf_tree.node.set(["env", key], val,
                                   state=conf_tree.node.STATE_NORMAL)
            conf_tree.node.set([jinja2_section, key], '"' + val + '"')

        # See if suite is running or not
        hosts = []
        if opts.host:
            hosts.append(opts.host)
        conf = ResourceLocator.default().get_conf()

        known_hosts = self.host_selector.expand(
              conf.get_value(["rose-suite-run", "hosts"], "").split() +
              conf.get_value(["rose-suite-run", "scan-hosts"], "").split() +
              ["localhost"])[0]
        known_hosts = list(set(known_hosts))

        for known_host in known_hosts:
#.........這裏部分代碼省略.........

class StemRunner(object):

    """Set up options for running a STEM job through Rose."""

    def __init__(self, opts, reporter=None, popen=None, fs_util=None):
        self.opts = opts
        if reporter is None:
            self.reporter = Reporter(opts.verbosity - opts.quietness)
        else:
            self.reporter = reporter
        if popen is None:
            self.popen = RosePopener(event_handler=self.reporter)
        else:
            self.popen = popen
        if fs_util is None:
            self.fs_util = FileSystemUtil(event_handler=self.reporter)
        else:
            self.fs_util = fs_util
        self.host_selector = HostSelector(event_handler=self.reporter,
                                          popen=self.popen)

    def _add_define_option(self, var, val):
        """Add a define option passed to the SuiteRunner."""

        if self.opts.defines:
            self.opts.defines.append(SUITE_RC_PREFIX + var + '=' + val)
        else:
            self.opts.defines = [SUITE_RC_PREFIX + var + '=' + val]
        self.reporter(ConfigVariableSetEvent(var, val))
        return

    def _get_base_dir(self, item):
        """Given a source tree return the following from 'fcm loc-layout':
           * url
           * sub_tree
           * peg_rev
           * root
           * project
        """

        rc, output, stderr = self.popen.run('fcm', 'loc-layout', item)
        if rc != 0:
            raise ProjectNotFoundException(item, stderr)

        ret = {}
        for line in output.splitlines():
            if ":" not in line:
                continue
            key, value = line.split(":", 1)
            if key:
                if value:
                    ret[key] = value.strip()

        return ret

    def _get_project_from_url(self, source_dict):
        """Run 'fcm keyword-print' to work out the project name."""

        repo = source_dict['root']
        if source_dict['project']:
            repo += '/' + source_dict['project']

        rc, kpoutput, stderr = self.popen.run('fcm', 'kp', source_dict['url'])

        project = None
        for line in kpoutput.splitlines():
            if line.rstrip().endswith(repo):
                kpresult = re.search(r'^location{primary}\[(.*)\]', line)
                if kpresult:
                    project = kpresult.group(1)
                    break
        return project

    def _deduce_mirror(self, source_dict, project):
        """Deduce the mirror location of this source tree."""

        # Root location for project
        proj_root = source_dict['root'] + '/' + source_dict['project']

        # Swap project to mirror
        project = re.sub(r'\.x$', r'.xm', project)
        mirror_repo = "fcm:" + project

        # Generate mirror location
        mirror = re.sub(proj_root, mirror_repo, source_dict['url'])

        # Remove any sub-tree
        mirror = re.sub(source_dict['sub_tree'], r'', mirror)
        mirror = re.sub(r'/@', r'@', mirror)

        # Add forwards slash after .xm if missing
        if '.xm/' not in mirror:
            mirror = re.sub(r'\.xm', r'.xm/', mirror)
        return mirror

    def _ascertain_project(self, item):
        """Set the project name and top-level from 'fcm loc-layout'.
        Returns:
            * project name
            * top-level location of the source tree with revision number
#.........這裏部分代碼省略.........

    def run(self, app_runner, conf_tree, opts, args, uuid, work_files):
        """Suite housekeeping application.

        This application is designed to work under "rose task-run" in a cycling
        suite.

        """
        suite_name = os.getenv("ROSE_SUITE_NAME")
        if not suite_name:
            return

        # Tar-gzip job logs on suite host
        # Prune job logs on remote hosts and suite host
        prune_remote_logs_cycles = self._get_conf(
            app_runner, conf_tree, "prune-remote-logs-at")
        prune_server_logs_cycles = self._get_conf(
            app_runner, conf_tree, "prune-server-logs-at")
        archive_logs_cycles = self._get_conf(
            app_runner, conf_tree, "archive-logs-at")
        if (prune_remote_logs_cycles or
                prune_server_logs_cycles or
                archive_logs_cycles):
            tmp_prune_remote_logs_cycles = []
            for cycle in prune_remote_logs_cycles:
                if cycle not in archive_logs_cycles:
                    tmp_prune_remote_logs_cycles.append(cycle)
            prune_remote_logs_cycles = tmp_prune_remote_logs_cycles

            tmp_prune_server_logs_cycles = []
            for cycle in prune_server_logs_cycles:
                if cycle not in archive_logs_cycles:
                    tmp_prune_server_logs_cycles.append(cycle)
            prune_server_logs_cycles = tmp_prune_server_logs_cycles

            if prune_remote_logs_cycles:
                app_runner.suite_engine_proc.job_logs_pull_remote(
                    suite_name, prune_remote_logs_cycles,
                    prune_remote_mode=True)

            if prune_server_logs_cycles:
                app_runner.suite_engine_proc.job_logs_remove_on_server(
                    suite_name, prune_server_logs_cycles)

            if archive_logs_cycles:
                app_runner.suite_engine_proc.job_logs_archive(
                    suite_name, archive_logs_cycles)

        # Prune other directories
        globs, cycle_set = self._get_prune_globs(app_runner, conf_tree)
        if not globs:
            return
        suite_engine_proc = app_runner.suite_engine_proc
        hosts = suite_engine_proc.get_suite_jobs_auths(
            suite_name, [(cycle, None) for cycle in cycle_set])
        # A shuffle here should allow the load for doing "rm -rf" to be shared
        # between job hosts who share a file system.
        shuffle(hosts)
        suite_dir_rel = suite_engine_proc.get_suite_dir_rel(suite_name)
        form_dict = {"d": suite_dir_rel, "g": " ".join(globs)}
        sh_cmd_head = r"set -e; cd %(d)s; " % form_dict
        sh_cmd = (
            r"set +e; ls -d %(g)s; " +
            r"set -e; rm -fr %(g)s") % form_dict
        cwd = os.getcwd()
        host_selector = HostSelector(
            app_runner.event_handler, app_runner.popen)
        for host in hosts + [host_selector.get_local_host()]:
            sdir = None
            try:
                if host_selector.is_local_host(host):
                    sdir = suite_engine_proc.get_suite_dir(suite_name)
                    app_runner.fs_util.chdir(sdir)
                    out = app_runner.popen.run_ok(
                        "bash", "-O", "extglob", "-c", sh_cmd)[0]
                else:
                    cmd = app_runner.popen.get_cmd(
                        "ssh", host,
                        "bash -O extglob -c '" + sh_cmd_head + sh_cmd + "'")
                    out = app_runner.popen.run_ok(*cmd)[0]
            except RosePopenError as exc:
                app_runner.handle_event(exc)
            else:
                if sdir is None:
                    event = FileSystemEvent(FileSystemEvent.CHDIR,
                                            host + ":" + suite_dir_rel)
                    app_runner.handle_event(event)
                for line in sorted(out.splitlines()):
                    if not host_selector.is_local_host(host):
                        line = host + ":" + line
                    event = FileSystemEvent(FileSystemEvent.DELETE, line)
                    app_runner.handle_event(event)
            finally:
                if sdir:
                    app_runner.fs_util.chdir(cwd)
        return

class SuiteRunner(Runner):

    """Invoke a Rose suite."""

    SLEEP_PIPE = 0.05
    NAME = "suite"
    OPTIONS = [
        "conf_dir",
        "defines",
        "defines_suite",
        "gcontrol_mode",
        "host",
        "install_only_mode",
        "local_install_only_mode",
        "log_archive_mode",
        "log_keep",
        "log_name",
        "name",
        "new_mode",
        "no_overwrite_mode",
        "opt_conf_keys",
        "reload_mode",
        "remote",
        "restart_mode",
        "run_mode",
        "strict_mode",
        "validate_suite_only"]

    # Lists of rsync (always) exclude globs
    SYNC_EXCLUDES = (
        "/.*",
        "/cylc-suite.db",
        "/log",
        "/log.*",
        "/state",
        "/share",
        "/work",
    )

    def __init__(self, *args, **kwargs):
        Runner.__init__(self, *args, **kwargs)
        self.host_selector = HostSelector(self.event_handler, self.popen)
        self.suite_run_cleaner = SuiteRunCleaner(
            event_handler=self.event_handler,
            host_selector=self.host_selector,
            suite_engine_proc=self.suite_engine_proc)

    def run_impl(self, opts, args, uuid, work_files):
        # Log file, temporary
        if hasattr(self.event_handler, "contexts"):
            t_file = TemporaryFile()
            log_context = ReporterContext(None, self.event_handler.VV, t_file)
            self.event_handler.contexts[uuid] = log_context

        # Check suite engine specific compatibility
        self.suite_engine_proc.check_global_conf_compat()

        # Suite name from the current working directory
        if opts.conf_dir:
            self.fs_util.chdir(opts.conf_dir)
        opts.conf_dir = os.getcwd()

        # --remote=KEY=VALUE,...
        if opts.remote:
            # opts.name always set for remote.
            return self._run_remote(opts, opts.name)

        conf_tree = self.config_load(opts)
        self.fs_util.chdir(conf_tree.conf_dirs[0])

        suite_name = opts.name
        if not opts.name:
            suite_name = os.path.basename(os.getcwd())

        # Check suite.rc #! line for template scheme
        templ_scheme = "jinja2"
        if self.suite_engine_proc.SUITE_CONF in conf_tree.files:
            suiterc_path = os.path.join(
                conf_tree.files[self.suite_engine_proc.SUITE_CONF],
                self.suite_engine_proc.SUITE_CONF)
            with open(suiterc_path) as fh:
                line = fh.readline()
                if line.startswith("#!"):
                    templ_scheme = line[2:].strip().lower()
        suite_section = (templ_scheme + ':' +
                         self.suite_engine_proc.SUITE_CONF)

        extra_defines = []
        if opts.defines_suite:
            for define in opts.defines_suite:
                extra_defines.append("[" + suite_section + "]" + define)

        # Automatic Rose constants
        # ROSE_ORIG_HOST: originating host
        # ROSE_VERSION: Rose version (not retained in run_mode=="reload")
        # Suite engine version
        my_rose_version = ResourceLocator.default().get_version()
        suite_engine_key = self.suite_engine_proc.get_version_env_name()
        if opts.run_mode in ["reload", "restart"]:
            prev_config_path = self.suite_engine_proc.get_suite_dir(
#.........這裏部分代碼省略.........

class RoseBushService(object):

    """Serves the index page."""

    NS = "rose"
    UTIL = "bush"
    TITLE = "Rose Bush"
    CYCLES_PER_PAGE = 100
    JOBS_PER_PAGE = 15
    JOBS_PER_PAGE_MAX = 300
    SUITES_PER_PAGE = 100
    VIEW_SIZE_MAX = 10 * 1024 * 1024  # 10MB

    def __init__(self, *args, **kwargs):
        self.exposed = True
        self.suite_engine_proc = SuiteEngineProcessor.get_processor()
        rose_conf = ResourceLocator.default().get_conf()
        self.logo = rose_conf.get_value(["rose-bush", "logo"])
        self.title = rose_conf.get_value(["rose-bush", "title"], self.TITLE)
        self.host_name = rose_conf.get_value(["rose-bush", "host"])
        if self.host_name is None:
            self.host_name = HostSelector().get_local_host()
            if self.host_name and "." in self.host_name:
                self.host_name = self.host_name.split(".", 1)[0]
        self.rose_version = ResourceLocator.default().get_version()
        template_env = jinja2.Environment(loader=jinja2.FileSystemLoader(
            ResourceLocator.default().get_util_home(
                "lib", "html", "template", "rose-bush")))

        def urlise(text):
            pattern = '((https?):\/\/[^\s\(\)&\[\]\{\}]+)'
            replacement = '<a href="\g<1>">\g<1></a>'
            text = re.sub(pattern, replacement, text)
            return text

        template_env.filters['urlise'] = urlise
        self.template_env = template_env

    @cherrypy.expose
    def index(self, form=None):
        """Display a page to input user ID and suite ID."""
        # TODO: some way to allow autocomplete of user field?
        data = {
            "logo": self.logo,
            "title": self.title,
            "host": self.host_name,
            "rose_version": self.rose_version,
            "script": cherrypy.request.script_name,
        }
        if form == "json":
            return simplejson.dumps(data)
        try:
            return self.template_env.get_template("index.html").render(**data)
        except Exception as exc:
            traceback.print_exc(exc)

    @cherrypy.expose
    def broadcast_states(self, user, suite, form=None):
        """List current broadcasts of a running or completed suite."""
        user_suite_dir = self._get_user_suite_dir(user, suite)
        data = {
            "logo": self.logo,
            "title": self.title,
            "host": self.host_name,
            "user": user,
            "suite": suite,
            "rose_version": self.rose_version,
            "script": cherrypy.request.script_name,
            "method": "broadcast_states",
            "states": {},
            "time": strftime("%Y-%m-%dT%H:%M:%S+0000", gmtime()),
        }
        data["states"].update(
            self.suite_engine_proc.get_suite_state_summary(user, suite))
        data.update(self._get_suite_logs_info(user, suite))
        data["broadcast_states"] = (
            self.suite_engine_proc.get_suite_broadcast_states(user, suite))
        if form == "json":
            return simplejson.dumps(data)
        try:
            template = self.template_env.get_template("broadcast-states.html")
            return template.render(**data)
        except Exception as exc:
            traceback.print_exc(exc)
        return simplejson.dumps(data)

    @cherrypy.expose
    def broadcast_events(self, user, suite, form=None):
        """List broadcasts history of a running or completed suite."""
        user_suite_dir = self._get_user_suite_dir(user, suite)
        data = {
            "logo": self.logo,
            "title": self.title,
            "host": self.host_name,
            "user": user,
            "suite": suite,
            "rose_version": self.rose_version,
            "script": cherrypy.request.script_name,
            "method": "broadcast_events",
            "states": {},
#.........這裏部分代碼省略.........