Python globalcfg.GLOBAL_CFG类代码示例

def main(name, start):
    # Parse the command line:
    server = start()

    # Print copyright and license information
    print_blurb()

    # Create run directory tree and get port.
    try:
        GLOBAL_CFG.create_cylc_run_tree(server.suite)
        server.configure_pyro()
    except Exception as exc:
        if flags.debug:
            raise
        else:
            sys.exit(exc)

    # Daemonize the suite
    if not server.options.no_detach and not flags.debug:
        daemonize(server)

    try:
        server.configure()
        server.run()
        # For profiling (see Python docs for how to display the stats).
        # import cProfile
        # cProfile.runctx('server.run()', globals(), locals(), 'stats')
    except SchedulerStop, x:
        # deliberate stop
        print str(x)
        server.shutdown()

    def __init__( self, suite ):

        sodir = GLOBAL_CFG.get_derived_host_item( suite, 'suite log directory' )
        self.opath = os.path.join( sodir, 'out' ) 
        self.epath = os.path.join( sodir, 'err' ) 

        # use same archive length as logging (TODO: document this)
        self.roll_at_startup = GLOBAL_CFG.get( ['suite logging','roll over at start-up'] )
        self.arclen = GLOBAL_CFG.get( ['suite logging','rolling archive length'] )

    def __init__( self, suite ):

        self.ldir = GLOBAL_CFG.get_derived_host_item( suite, 'suite log directory' )
        self.path = os.path.join( self.ldir, 'log' ) 

        self.err_path = os.path.join( self.ldir, 'err' )
        self.roll_at_startup = GLOBAL_CFG.get( ['suite logging','roll over at start-up'] )
        self.n_keep = GLOBAL_CFG.get( ['suite logging','rolling archive length'] )
        self.max_bytes = GLOBAL_CFG.get( ['suite logging','maximum size in bytes'] )

 def _load_remote_item(self, item, reg, owner, host):
     """Load content of service item from remote [[email protected]]host via SSH."""
     if not is_remote_host(host) and not is_remote_user(owner):
         return
     # Prefix STDOUT to ensure returned content is relevant
     prefix = r'[CYLC-AUTH] %(suite)s' % {'suite': reg}
     # Attempt to cat passphrase file under suite service directory
     from cylc.cfgspec.globalcfg import GLOBAL_CFG
     script = (
         r"""echo '%(prefix)s'; """
         r'''cat "%(run_d)s/%(srv_base)s/%(item)s"'''
     ) % {
         'prefix': prefix,
         'run_d': GLOBAL_CFG.get_derived_host_item(
             reg, 'suite run directory', host, owner),
         'srv_base': self.DIR_BASE_SRV,
         'item': item
     }
     import shlex
     command = shlex.split(
         GLOBAL_CFG.get_host_item('ssh command', host, owner))
     command += ['-n', owner + '@' + host, script]
     from subprocess import Popen, PIPE
     try:
         proc = Popen(command, stdout=PIPE, stderr=PIPE)
     except OSError:
         if cylc.flags.debug:
             import traceback
             traceback.print_exc()
         return
     out, err = proc.communicate()
     ret_code = proc.wait()
     # Extract passphrase from STDOUT
     # It should live in the line with the correct prefix
     content = ""
     can_read = False
     for line in out.splitlines(True):
         if can_read:
             content += line
         elif line.strip() == prefix:
             can_read = True
     if not content or ret_code:
         if cylc.flags.debug:
             print >> sys.stderr, (
                 'ERROR: %(command)s # code=%(ret_code)s\n%(err)s\n'
             ) % {
                 'command': command,
                 # STDOUT may contain passphrase, so not safe to print
                 # 'out': out,
                 'err': err,
                 'ret_code': ret_code,
             }
         return
     return content

    def init_suite_run_dir(self, suite_name, user_at_host):
        """Initialise suite run dir on a [email protected]

        Create SUITE_RUN_DIR/log/job/ if necessary.
        Install suite contact environment file.
        Install suite python modules.

        Raise RemoteJobHostInitError if initialisation cannot complete.

        """
        if '@' in user_at_host:
            owner, host = user_at_host.split('@', 1)
        else:
            owner, host = None, user_at_host
        if ((owner, host) in [(None, 'localhost'), (USER, 'localhost')] or
                host in self.initialised_hosts or
                self.single_task_mode):
            return

        suite_run_dir = GLOBAL_CFG.get_derived_host_item(
            suite_name, 'suite run directory')
        sources = [os.path.join(suite_run_dir, CylcSuiteEnv.BASE_NAME)]
        if 'CYLC_SUITE_DEF_PATH' in os.environ:
            sources.append(
                os.path.join(os.getenv('CYLC_SUITE_DEF_PATH'), 'passphrase'))
        suite_run_py = os.path.join(suite_run_dir, 'python')
        if os.path.isdir(suite_run_py):
            sources.append(suite_run_py)
        r_suite_run_dir = GLOBAL_CFG.get_derived_host_item(
            suite_name, 'suite run directory', host, owner)
        r_log_job_dir = GLOBAL_CFG.get_derived_host_item(
            suite_name, 'suite job log directory', host, owner)
        getLogger('main').log(INFO, 'Initialising %s:%s' % (
            user_at_host, r_suite_run_dir))

        ssh_tmpl = GLOBAL_CFG.get_host_item(
            'remote shell template', host, owner).replace(' %s', '')
        scp_tmpl = GLOBAL_CFG.get_host_item(
            'remote copy template', host, owner)

        cmd1 = shlex.split(ssh_tmpl) + [
            "-n", user_at_host,
            'mkdir', '-p', r_suite_run_dir, r_log_job_dir]
        cmd2 = shlex.split(scp_tmpl) + ['-pr'] + sources + [
            user_at_host + ":" + r_suite_run_dir + '/']
        for cmd in [cmd1, cmd2]:
            proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
            out, err = proc.communicate()
            if proc.wait():
                raise RemoteJobHostInitError(
                    user_at_host, " ".join([quote(item) for item in cmd]),
                    proc.returncode, out, err)
        self.initialised_hosts.append(user_at_host)

 def __init__(self, suite, run_mode='live', ict=None, stop_point=None):
     self.run_mode = run_mode
     self.cts_str = None
     self.set_cts(ict, stop_point)
     self.dir_name = GLOBAL_CFG.get_derived_host_item(
         suite, 'suite state directory')
     self.file_name = os.path.join(self.dir_name, self.BASE_NAME)
     self.arch_len = GLOBAL_CFG.get(['state dump rolling archive length'])
     if not self.arch_len or int(self.arch_len) <= 1:
         self.arch_len = 1
     self.arch_files = []
     self.pool = None
     self.log = logging.getLogger('main')

    def init_suite_run_dir(self, suite_name, user_at_host):
        """Initialise suite run dir on a [email protected]

        Create SUITE_RUN_DIR/log/job/ if necessary.
        Install suite contact environment file.
        Install suite python modules.

        Raise RemoteJobHostInitError if initialisation cannot complete.

        """
        if '@' in user_at_host:
            owner, host = user_at_host.split('@', 1)
        else:
            owner, host = None, user_at_host
        if ((owner, host) in [(None, 'localhost'), (user, 'localhost')] or
                host in self.initialised_hosts or
                self.single_task_mode):
            return

        suite_run_dir = GLOBAL_CFG.get_derived_host_item(
            suite_name, 'suite run directory')
        sources = [os.path.join(suite_run_dir, "cylc-suite-env")]
        suite_run_py = os.path.join(suite_run_dir, "python")
        if os.path.isdir(suite_run_py):
            sources.append(suite_run_py)
        try:
            r_suite_run_dir = GLOBAL_CFG.get_derived_host_item(
                suite_name, 'suite run directory', host, owner)
            r_log_job_dir = GLOBAL_CFG.get_derived_host_item(
                suite_name, 'suite job log directory', host, owner)
            getLogger('main').log(INFO, 'Initialising %s:%s' % (
                user_at_host, r_suite_run_dir))

            ssh_tmpl = GLOBAL_CFG.get_host_item(
                'remote shell template', host, owner).replace(" %s", "")
            scp_tmpl = GLOBAL_CFG.get_host_item(
                'remote copy template', host, owner)

            cmd1 = shlex.split(ssh_tmpl) + [
                user_at_host,
                'mkdir -p "%s" "%s"' % (r_suite_run_dir, r_log_job_dir)]
            cmd2 = shlex.split(scp_tmpl) + ["-r"] + sources + [
                user_at_host + ":" + r_suite_run_dir + "/"]
            for cmd in [cmd1, cmd2]:
                check_call(cmd)
        except Exception:
            raise RemoteJobHostInitError(user_at_host)
        self.initialised_hosts.append(user_at_host)

    def unlink_hosts_contacts(self, reg):
        """Remove suite contact files from initialised hosts.

        This is called on shutdown, so we don't want anything to hang.
        Terminate any incomplete SSH commands after 10 seconds.
        """
        # Issue all SSH commands in parallel
        procs = {}
        for (host, owner), should_unlink in self.init_host_map.items():
            if not should_unlink:
                continue
            user_at_host = host
            if owner:
                user_at_host = owner + '@' + host
            ssh_tmpl = GLOBAL_CFG.get_host_item('ssh command', host, owner)
            r_suite_contact_file = os.path.join(
                GLOBAL_CFG.get_derived_host_item(
                    reg, 'suite run directory', host, owner),
                self.suite_srv_files_mgr.DIR_BASE_SRV,
                self.suite_srv_files_mgr.FILE_BASE_CONTACT)
            cmd = shlex.split(ssh_tmpl) + [
                '-n', user_at_host, 'rm', '-f', r_suite_contact_file]
            procs[user_at_host] = (cmd, Popen(cmd, stdout=PIPE, stderr=PIPE))
        # Wait for commands to complete for a max of 10 seconds
        timeout = time() + 10.0
        while procs and time() < timeout:
            for user_at_host, (cmd, proc) in procs.copy().items():
                if proc.poll() is None:
                    continue
                del procs[user_at_host]
                out, err = proc.communicate()
                if proc.wait():
                    ERR.warning(RemoteJobHostInitError(
                        RemoteJobHostInitError.MSG_TIDY,
                        user_at_host, ' '.join([quote(item) for item in cmd]),
                        proc.returncode, out, err))
        # Terminate any remaining commands
        for user_at_host, (cmd, proc) in procs.items():
            try:
                proc.terminate()
            except OSError:
                pass
            out, err = proc.communicate()
            if proc.wait():
                ERR.warning(RemoteJobHostInitError(
                    RemoteJobHostInitError.MSG_TIDY,
                    user_at_host, ' '.join([quote(item) for item in cmd]),
                    proc.returncode, out, err))

def get_port( suite, owner=user, host=get_hostname(), pphrase=None, pyro_timeout=None ):
    # Scan ports until a particular suite is found.

    pyro_base_port = GLOBAL_CFG.get( ['pyro','base port'] )
    pyro_port_range = GLOBAL_CFG.get( ['pyro','maximum number of ports'] )

    for port in range( pyro_base_port, pyro_base_port + pyro_port_range ):
        uri = cylcid_uri( host, port )
        try:
            proxy = Pyro.core.getProxyForURI(uri)
        except Pyro.errors.URIError, x:
            # No such host?
            raise SuiteNotFoundError, x

        if pyro_timeout: # convert from string
            pyro_timeout = float( pyro_timeout )

        proxy._setTimeout(pyro_timeout)
        proxy._setIdentification( pphrase )

        before = datetime.datetime.now()
        try:
            name, xowner = proxy.id()
        except Pyro.errors.TimeoutError:
            warn_timeout( host, port, pyro_timeout )
            pass
        except Pyro.errors.ConnectionDeniedError:
            #print >> sys.stderr, "Wrong suite or wrong passphrase at " + portid( host, port )
            pass
        except Pyro.errors.ProtocolError:
            #print >> sys.stderr, "No Suite Found at " + portid( host, port )
            pass
        except Pyro.errors.NamingError:
            #print >> sys.stderr, "Non-cylc pyro server found at " + portid( host, port )
            pass
        else:
            if flags.verbose:
                after = datetime.datetime.now()
                print "Pyro connection on port " +str(port) + " took: " + str( after - before )
            if name == suite and xowner == owner:
                if flags.verbose:
                    print suite, owner, host, port
                # RESULT
                return port
            else:
                # ID'd some other suite.
                #print 'OTHER SUITE:', name, xowner, host, port
                pass

 def unregister(self, exp):
     """Un-register a suite."""
     unregistered_set = set()
     skipped_set = set()
     ports_d = GLOBAL_CFG.get(['pyro', 'ports directory'])
     for name in sorted(self.list_all_suites()):
         if not re.match(exp + r'\Z', name):
             continue
         try:
             data = self.get_suite_data(name)
         except RegistrationError:
             continue
         if os.path.exists(os.path.join(ports_d, name)):
             skipped_set.add((name, data['path']))
             print >> sys.stderr, (
                 'SKIP UNREGISTER %s: port file exists' % (name))
             continue
         for base_name in ['passphrase', 'suite.rc.processed']:
             try:
                 os.unlink(os.path.join(data['path'], base_name))
             except OSError:
                 pass
         unregistered_set.add((name, data['path']))
         print 'UNREGISTER %s:%s' % (name, data['path'])
         os.unlink(os.path.join(self.dbpath, name))
     return unregistered_set, skipped_set

def prompt(question, force=False, gui=False, no_force=False, no_abort=False):
    """Interactive Yes/No prompt for cylc CLI scripts.

    For convenience, on No we just exit rather than return.
    If force is True don't prompt, just return immediately.

    """
    if (force or GLOBAL_CFG.get(['disable interactive command prompts'])) and (
            not no_force):
        return True
    if gui:
        import gtk
        dialog = gtk.MessageDialog(
            None, gtk.DIALOG_DESTROY_WITH_PARENT,
            gtk.MESSAGE_QUESTION, gtk.BUTTONS_YES_NO,
            question
        )
        gui_response = dialog.run()
        response_no = (gui_response != gtk.RESPONSE_YES)
    else:
        cli_response = raw_input('%s (y/n)? ' % question)
        response_no = (cli_response not in ['y', 'Y'])
    if response_no:
        if no_abort:
            return False
        else:
            sys.exit(0)
    else:
        return True

    def get_create_job_log_path(cls, suite, task_name, task_point, submit_num):
        """Return a new job log path on the suite host, in two parts.

        /part1/part2

        * part1: the top level job log directory on the suite host.
        * part2: the rest, which is also used on remote task hosts.

        The full local job log directory is created if necessary, and its
        parent symlinked to NN (submit number).

        """

        suite_job_log_dir = GLOBAL_CFG.get_derived_host_item(
            suite, "suite job log directory")

        the_rest_dir = os.path.join(
            str(task_point), task_name, "%02d" % int(submit_num))
        the_rest = os.path.join(the_rest_dir, "job")

        local_log_dir = os.path.join(suite_job_log_dir, the_rest_dir)

        mkdir_p(local_log_dir)
        target = os.path.join(os.path.dirname(local_log_dir), "NN")
        try:
            os.unlink(target)
        except OSError:
            pass
        try:
            os.symlink(os.path.basename(local_log_dir), target)
        except OSError as exc:
            if not exc.filename:
                exc.filename = target
            raise exc
        return suite_job_log_dir, the_rest

 def __init__(self, suite, host, owner):
     self.suite = suite
     self.host = host
     self.owner = owner
     self.locn = None
     self.local_path = os.path.join(
         GLOBAL_CFG.get(['pyro', 'ports directory']), suite)

 def __init__(self, hosts=None, owner=None, poll_interval=None,
              is_compact=False):
     # We can't use gobject.threads_init() for panel applets.
     warnings.filterwarnings('ignore', 'use the new', Warning)
     setup_icons()
     if not hosts:
         try:
             hosts = GLOBAL_CFG.get(["suite host scanning", "hosts"])
         except KeyError:
             hosts = ["localhost"]
     self.is_compact = is_compact
     self.hosts = hosts
     if owner is None:
         owner = user
     self.owner = owner
     dot_hbox = gtk.HBox()
     dot_hbox.show()
     dot_eb = gtk.EventBox()
     dot_eb.show()
     dot_eb.add(dot_hbox)
     image = gtk.image_new_from_stock("gcylc", gtk.ICON_SIZE_MENU)
     image.show()
     image_eb = gtk.EventBox()
     image_eb.show()
     image_eb.connect("button-press-event", self._on_button_press_event)
     image_eb.add(image)
     self.top_hbox = gtk.HBox()
     self.top_hbox.pack_start(image_eb, expand=False, fill=False)
     self.top_hbox.pack_start(dot_eb, expand=False, fill=False, padding=2)
     self.top_hbox.show()
     self.updater = ScanPanelAppletUpdater(hosts, dot_hbox, image,
                                           self.is_compact,
                                           owner=owner,
                                           poll_interval=poll_interval)
     self.top_hbox.connect("destroy", self.stop)

def handle_proxies():
    """Unset proxies if the configuration matches this."""
    from cylc.cfgspec.globalcfg import GLOBAL_CFG
    if not GLOBAL_CFG.get(['communication', 'proxies on']):
        import os
        os.environ.pop("http_proxy", None)
        os.environ.pop("https_proxy", None)