Python hub.HubFactory类代码示例

 def init_hub(self):
     c = self.config
     
     self.do_import_statements()
     
     try:
         self.factory = HubFactory(config=c, log=self.log)
         # self.start_logging()
         self.factory.init_hub()
     except TraitError:
         raise
     except Exception:
         self.log.error("Couldn't construct the Controller", exc_info=True)
         self.exit(1)
     
     if self.write_connection_files:
         # save to new json config files
         f = self.factory
         cdict = {'exec_key' : f.session.key.decode('ascii'),
                 'ssh' : self.ssh_server,
                 'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
                 'location' : self.location
                 }
         self.save_connection_dict(self.client_json_file, cdict)
         edict = cdict
         edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
         edict['ssh'] = self.engine_ssh_server
         self.save_connection_dict(self.engine_json_file, edict)

 def init_hub(self):
     c = self.config
     
     self.do_import_statements()
     reusing = self.reuse_files
     if reusing:
         try:
             self.load_config_from_json()
         except (AssertionError,IOError):
             reusing=False
     # check again, because reusing may have failed:
     if reusing:
         pass
     elif self.secure:
         key = str(uuid.uuid4())
         # keyfile = os.path.join(self.profile_dir.security_dir, self.exec_key)
         # with open(keyfile, 'w') as f:
         #     f.write(key)
         # os.chmod(keyfile, stat.S_IRUSR|stat.S_IWUSR)
         c.Session.key = asbytes(key)
     else:
         key = c.Session.key = b''
     
     try:
         self.factory = HubFactory(config=c, log=self.log)
         # self.start_logging()
         self.factory.init_hub()
     except:
         self.log.error("Couldn't construct the Controller", exc_info=True)
         self.exit(1)
     
     if not reusing:
         # save to new json config files
         f = self.factory
         cdict = {'exec_key' : key,
                 'ssh' : self.ssh_server,
                 'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
                 'location' : self.location
                 }
         self.save_connection_dict('ipcontroller-client.json', cdict)
         edict = cdict
         edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
         edict['ssh'] = self.engine_ssh_server
         self.save_connection_dict('ipcontroller-engine.json', edict)

    def init_hub(self):
        c = self.config

        self.do_import_statements()

        try:
            self.factory = HubFactory(config=c, log=self.log)
            # self.start_logging()
            self.factory.init_hub()
        except TraitError:
            raise
        except Exception:
            self.log.error("Couldn't construct the Controller", exc_info=True)
            self.exit(1)

        if self.write_connection_files:
            # save to new json config files
            f = self.factory
            base = {
                "key": f.session.key.decode("ascii"),
                "location": self.location,
                "pack": f.session.packer,
                "unpack": f.session.unpacker,
                "signature_scheme": f.session.signature_scheme,
            }

            cdict = {"ssh": self.ssh_server}
            cdict.update(f.client_info)
            cdict.update(base)
            self.save_connection_dict(self.client_json_file, cdict)

            edict = {"ssh": self.engine_ssh_server}
            edict.update(f.engine_info)
            edict.update(base)
            self.save_connection_dict(self.engine_json_file, edict)

        fname = "engines%s.json" % self.cluster_id
        self.factory.hub.engine_state_file = os.path.join(self.profile_dir.log_dir, fname)
        if self.restore_engines:
            self.factory.hub._load_engine_state()
        # load key into config so other sessions in this process (TaskScheduler)
        # have the same value
        self.config.Session.key = self.factory.session.key

    def init_hub(self):
        c = self.config

        self.do_import_statements()

        try:
            self.factory = HubFactory(config=c, log=self.log)
            # self.start_logging()
            self.factory.init_hub()
        except TraitError:
            raise
        except Exception:
            self.log.error("Couldn't construct the Controller", exc_info=True)
            self.exit(1)

        if self.write_connection_files:
            # save to new json config files
            f = self.factory
            base = {
                'key': f.session.key.decode('ascii'),
                'location': self.location,
                'pack': f.session.packer,
                'unpack': f.session.unpacker,
                'signature_scheme': f.session.signature_scheme,
            }

            cdict = {'ssh': self.ssh_server}
            cdict.update(f.client_info)
            cdict.update(base)
            self.save_connection_dict(self.client_json_file, cdict)

            edict = {'ssh': self.engine_ssh_server}
            edict.update(f.engine_info)
            edict.update(base)
            self.save_connection_dict(self.engine_json_file, edict)

        fname = "engines%s.json" % self.cluster_id
        self.factory.hub.engine_state_file = os.path.join(
            self.profile_dir.log_dir, fname)
        if self.restore_engines:
            self.factory.hub._load_engine_state()

class IPControllerApp(BaseParallelApplication):

    name = u'ipcontroller'
    description = _description
    examples = _examples
    config_file_name = Unicode(default_config_file_name)
    classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo
    
    # change default to True
    auto_create = Bool(True, config=True,
        help="""Whether to create profile dir if it doesn't exist.""")
    
    reuse_files = Bool(False, config=True,
        help="""Whether to reuse existing json connection files.
        If False, connection files will be removed on a clean exit.
        """
    )
    ssh_server = Unicode(u'', config=True,
        help="""ssh url for clients to use when connecting to the Controller
        processes. It should be of the form: [[email protected]]server[:port]. The
        Controller's listening addresses must be accessible from the ssh server""",
    )
    engine_ssh_server = Unicode(u'', config=True,
        help="""ssh url for engines to use when connecting to the Controller
        processes. It should be of the form: [[email protected]]server[:port]. The
        Controller's listening addresses must be accessible from the ssh server""",
    )
    location = Unicode(u'', config=True,
        help="""The external IP or domain name of the Controller, used for disambiguating
        engine and client connections.""",
    )
    import_statements = List([], config=True,
        help="import statements to be run at startup.  Necessary in some environments"
    )

    use_threads = Bool(False, config=True,
        help='Use threads instead of processes for the schedulers',
    )

    engine_json_file = Unicode('ipcontroller-engine.json', config=True,
        help="JSON filename where engine connection info will be stored.")
    client_json_file = Unicode('ipcontroller-client.json', config=True,
        help="JSON filename where client connection info will be stored.")
    
    def _cluster_id_changed(self, name, old, new):
        super(IPControllerApp, self)._cluster_id_changed(name, old, new)
        self.engine_json_file = "%s-engine.json" % self.name
        self.client_json_file = "%s-client.json" % self.name


    # internal
    children = List()
    mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')

    def _use_threads_changed(self, name, old, new):
        self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')
    
    write_connection_files = Bool(True,
        help="""Whether to write connection files to disk.
        True in all cases other than runs with `reuse_files=True` *after the first*
        """
    )

    aliases = Dict(aliases)
    flags = Dict(flags)
    

    def save_connection_dict(self, fname, cdict):
        """save a connection dict to json file."""
        c = self.config
        url = cdict['url']
        location = cdict['location']
        if not location:
            try:
                proto,ip,port = split_url(url)
            except AssertionError:
                pass
            else:
                try:
                    location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
                except (socket.gaierror, IndexError):
                    self.log.warn("Could not identify this machine's IP, assuming 127.0.0.1."
                    " You may need to specify '--location=<external_ip_address>' to help"
                    " IPython decide when to connect via loopback.")
                    location = '127.0.0.1'
            cdict['location'] = location
        fname = os.path.join(self.profile_dir.security_dir, fname)
        self.log.info("writing connection info to %s", fname)
        with open(fname, 'w') as f:
            f.write(json.dumps(cdict, indent=2))
        os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
    
    def load_config_from_json(self):
        """load config from existing json connector files."""
        c = self.config
        self.log.debug("loading config from JSON")
        # load from engine config
        fname = os.path.join(self.profile_dir.security_dir, self.engine_json_file)
        self.log.info("loading connection info from %s", fname)
        with open(fname) as f:
#.........这里部分代码省略.........

class IPControllerApp(BaseParallelApplication):

    name = 'ipcontroller'
    description = _description
    examples = _examples
    config_file_name = Unicode(default_config_file_name)
    classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, SQLiteDB] + maybe_mongo
    
    # change default to True
    auto_create = Bool(True, config=True,
        help="""Whether to create profile dir if it doesn't exist.""")
    
    reuse_files = Bool(False, config=True,
        help='Whether to reuse existing json connection files.'
    )
    secure = Bool(True, config=True,
        help='Whether to use HMAC digests for extra message authentication.'
    )
    ssh_server = Unicode('', config=True,
        help="""ssh url for clients to use when connecting to the Controller
        processes. It should be of the form: [[email protected]]server[:port]. The
        Controller's listening addresses must be accessible from the ssh server""",
    )
    location = Unicode('', config=True,
        help="""The external IP or domain name of the Controller, used for disambiguating
        engine and client connections.""",
    )
    import_statements = List([], config=True,
        help="import statements to be run at startup.  Necessary in some environments"
    )

    use_threads = Bool(False, config=True,
        help='Use threads instead of processes for the schedulers',
        )

    # internal
    children = List()
    mq_class = Unicode('zmq.devices.ProcessMonitoredQueue')

    def _use_threads_changed(self, name, old, new):
        self.mq_class = 'zmq.devices.%sMonitoredQueue'%('Thread' if new else 'Process')

    aliases = Dict(aliases)
    flags = Dict(flags)
    

    def save_connection_dict(self, fname, cdict):
        """save a connection dict to json file."""
        c = self.config
        url = cdict['url']
        location = cdict['location']
        if not location:
            try:
                proto,ip,port = split_url(url)
            except AssertionError:
                pass
            else:
                try:
                    location = socket.gethostbyname_ex(socket.gethostname())[2][-1]
                except (socket.gaierror, IndexError):
                    self.log.warn("Could not identify this machine's IP, assuming 127.0.0.1."
                    " You may need to specify '--location=<external_ip_address>' to help"
                    " IPython decide when to connect via loopback.")
                    location = '127.0.0.1'
            cdict['location'] = location
        fname = os.path.join(self.profile_dir.security_dir, fname)
        with open(fname, 'wb') as f:
            f.write(json.dumps(cdict, indent=2))
        os.chmod(fname, stat.S_IRUSR|stat.S_IWUSR)
    
    def load_config_from_json(self):
        """load config from existing json connector files."""
        c = self.config
        # load from engine config
        with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-engine.json')) as f:
            cfg = json.loads(f.read())
        key = c.Session.key = asbytes(cfg['exec_key'])
        xport,addr = cfg['url'].split('://')
        c.HubFactory.engine_transport = xport
        ip,ports = addr.split(':')
        c.HubFactory.engine_ip = ip
        c.HubFactory.regport = int(ports)
        self.location = cfg['location']
        # load client config
        with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-client.json')) as f:
            cfg = json.loads(f.read())
        assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
        xport,addr = cfg['url'].split('://')
        c.HubFactory.client_transport = xport
        ip,ports = addr.split(':')
        c.HubFactory.client_ip = ip
        self.ssh_server = cfg['ssh']
        assert int(ports) == c.HubFactory.regport, "regport mismatch"
    
    def init_hub(self):
        c = self.config
        
        self.do_import_statements()
        reusing = self.reuse_files
        if reusing:
#.........这里部分代码省略.........

class IPControllerApp(BaseParallelApplication):

    name = u"ipcontroller"
    description = _description
    examples = _examples
    config_file_name = Unicode(default_config_file_name)
    classes = [ProfileDir, Session, HubFactory, TaskScheduler, HeartMonitor, DictDB] + real_dbs

    # change default to True
    auto_create = Bool(True, config=True, help="""Whether to create profile dir if it doesn't exist.""")

    reuse_files = Bool(
        False,
        config=True,
        help="""Whether to reuse existing json connection files.
        If False, connection files will be removed on a clean exit.
        """,
    )
    restore_engines = Bool(
        False,
        config=True,
        help="""Reload engine state from JSON file
        """,
    )
    ssh_server = Unicode(
        u"",
        config=True,
        help="""ssh url for clients to use when connecting to the Controller
        processes. It should be of the form: [[email protected]]server[:port]. The
        Controller's listening addresses must be accessible from the ssh server""",
    )
    engine_ssh_server = Unicode(
        u"",
        config=True,
        help="""ssh url for engines to use when connecting to the Controller
        processes. It should be of the form: [[email protected]]server[:port]. The
        Controller's listening addresses must be accessible from the ssh server""",
    )
    location = Unicode(
        u"",
        config=True,
        help="""The external IP or domain name of the Controller, used for disambiguating
        engine and client connections.""",
    )
    import_statements = List(
        [], config=True, help="import statements to be run at startup.  Necessary in some environments"
    )

    use_threads = Bool(False, config=True, help="Use threads instead of processes for the schedulers")

    engine_json_file = Unicode(
        "ipcontroller-engine.json", config=True, help="JSON filename where engine connection info will be stored."
    )
    client_json_file = Unicode(
        "ipcontroller-client.json", config=True, help="JSON filename where client connection info will be stored."
    )

    def _cluster_id_changed(self, name, old, new):
        super(IPControllerApp, self)._cluster_id_changed(name, old, new)
        self.engine_json_file = "%s-engine.json" % self.name
        self.client_json_file = "%s-client.json" % self.name

    # internal
    children = List()
    mq_class = Unicode("zmq.devices.ProcessMonitoredQueue")

    def _use_threads_changed(self, name, old, new):
        self.mq_class = "zmq.devices.%sMonitoredQueue" % ("Thread" if new else "Process")

    write_connection_files = Bool(
        True,
        help="""Whether to write connection files to disk.
        True in all cases other than runs with `reuse_files=True` *after the first*
        """,
    )

    aliases = Dict(aliases)
    flags = Dict(flags)

    def save_connection_dict(self, fname, cdict):
        """save a connection dict to json file."""
        c = self.config
        url = cdict["registration"]
        location = cdict["location"]

        if not location:
            if PUBLIC_IPS:
                location = PUBLIC_IPS[-1]
            else:
                self.log.warn(
                    "Could not identify this machine's IP, assuming %s."
                    " You may need to specify '--location=<external_ip_address>' to help"
                    " IPython decide when to connect via loopback." % LOCALHOST
                )
                location = LOCALHOST
            cdict["location"] = location
        fname = os.path.join(self.profile_dir.security_dir, fname)
        self.log.info("writing connection info to %s", fname)
        with open(fname, "w") as f:
            f.write(json.dumps(cdict, indent=2))
#.........这里部分代码省略.........