Python engine.EngineFactory类代码示例

    def init_engine(self):
        # This is the working dir by now.
        sys.path.insert(0, "")
        config = self.config
        # print config
        self.find_url_file()

        # was the url manually specified?
        keys = set(self.config.EngineFactory.keys())
        keys = keys.union(set(self.config.RegistrationFactory.keys()))

        if keys.intersection(set(["ip", "url", "port"])):
            # Connection info was specified, don't wait for the file
            url_specified = True
            self.wait_for_url_file = 0
        else:
            url_specified = False

        if self.wait_for_url_file and not os.path.exists(self.url_file):
            self.log.warn("url_file %r not found", self.url_file)
            self.log.warn("Waiting up to %.1f seconds for it to arrive.", self.wait_for_url_file)
            tic = time.time()
            while not os.path.exists(self.url_file) and (time.time() - tic < self.wait_for_url_file):
                # wait for url_file to exist, or until time limit
                time.sleep(0.1)

        if os.path.exists(self.url_file):
            self.load_connector_file()
        elif not url_specified:
            self.log.fatal("Fatal: url file never arrived: %s", self.url_file)
            self.exit(1)

        try:
            exec_lines = config.IPKernelApp.exec_lines
        except AttributeError:
            try:
                exec_lines = config.InteractiveShellApp.exec_lines
            except AttributeError:
                exec_lines = config.IPKernelApp.exec_lines = []
        try:
            exec_files = config.IPKernelApp.exec_files
        except AttributeError:
            try:
                exec_files = config.InteractiveShellApp.exec_files
            except AttributeError:
                exec_files = config.IPKernelApp.exec_files = []

        if self.startup_script:
            exec_files.append(self.startup_script)
        if self.startup_command:
            exec_lines.append(self.startup_command)

        # Create the underlying shell class and Engine
        # shell_class = import_item(self.master_config.Global.shell_class)
        # print self.config
        try:
            self.engine = EngineFactory(config=config, log=self.log, connection_info=self.connection_info)
        except:
            self.log.error("Couldn't start the Engine", exc_info=True)
            self.exit(1)

    def construct(self):
        # This is the working dir by now.
        sys.path.insert(0, '')
        config = self.master_config
        # if os.path.exists(config.Global.key_file) and config.Global.secure:
        #     config.SessionFactory.exec_key = config.Global.key_file
        if os.path.exists(config.Global.url_file):
            with open(config.Global.url_file) as f:
                d = json.loads(f.read())
                for k,v in d.iteritems():
                    if isinstance(v, unicode):
                        d[k] = v.encode()
            if d['exec_key']:
                config.SessionFactory.exec_key = d['exec_key']
            d['url'] = disambiguate_url(d['url'], d['location'])
            config.RegistrationFactory.url=d['url']
            config.EngineFactory.location = d['location']
        
        
        
        config.Kernel.exec_lines = config.Global.exec_lines

        self.start_mpi()

        # Create the underlying shell class and EngineService
        # shell_class = import_item(self.master_config.Global.shell_class)
        try:
            self.engine = EngineFactory(config=config, logname=self.log.name)
        except:
            self.log.error("Couldn't start the Engine", exc_info=True)
            self.exit(1)
        
        self.start_logging()

    def init_engine(self):
        # This is the working dir by now.
        sys.path.insert(0, '')
        config = self.config
        # print config
        self.find_url_file()
        
        # was the url manually specified?
        keys = set(self.config.EngineFactory.keys())
        keys = keys.union(set(self.config.RegistrationFactory.keys()))
        
        if keys.intersection(set(['ip', 'url', 'port'])):
            # Connection info was specified, don't wait for the file
            url_specified = True
            self.wait_for_url_file = 0
        else:
            url_specified = False

        if self.wait_for_url_file and not os.path.exists(self.url_file):
            self.log.warn("url_file %r not found"%self.url_file)
            self.log.warn("Waiting up to %.1f seconds for it to arrive."%self.wait_for_url_file)
            tic = time.time()
            while not os.path.exists(self.url_file) and (time.time()-tic < self.wait_for_url_file):
                # wait for url_file to exist, for up to 10 seconds
                time.sleep(0.1)
            
        if os.path.exists(self.url_file):
            self.load_connector_file()
        elif not url_specified:
            self.log.critical("Fatal: url file never arrived: %s"%self.url_file)
            self.exit(1)
        
        
        try:
            exec_lines = config.Kernel.exec_lines
        except AttributeError:
            config.Kernel.exec_lines = []
            exec_lines = config.Kernel.exec_lines
        
        if self.startup_script:
            enc = sys.getfilesystemencoding() or 'utf8'
            cmd="execfile(%r)"%self.startup_script.encode(enc)
            exec_lines.append(cmd)
        if self.startup_command:
            exec_lines.append(self.startup_command)

        # Create the underlying shell class and Engine
        # shell_class = import_item(self.master_config.Global.shell_class)
        # print self.config
        try:
            self.engine = EngineFactory(config=config, log=self.log)
        except:
            self.log.error("Couldn't start the Engine", exc_info=True)
            self.exit(1)

    def init_engine(self):
        # This is the working dir by now.
        sys.path.insert(0, '')
        config = self.config
        # print config
        self.find_url_file()

        # if os.path.exists(config.Global.key_file) and config.Global.secure:
        #     config.SessionFactory.exec_key = config.Global.key_file
        if os.path.exists(self.url_file):
            with open(self.url_file) as f:
                d = json.loads(f.read())
                for k,v in d.items():
                    if isinstance(v, str):
                        d[k] = v.encode()
            if d['exec_key']:
                config.Session.key = d['exec_key']
            d['url'] = disambiguate_url(d['url'], d['location'])
            config.EngineFactory.url = d['url']
            config.EngineFactory.location = d['location']
        
        try:
            exec_lines = config.Kernel.exec_lines
        except AttributeError:
            config.Kernel.exec_lines = []
            exec_lines = config.Kernel.exec_lines
        
        if self.startup_script:
            enc = sys.getfilesystemencoding() or 'utf8'
            cmd="execfile(%r)"%self.startup_script.encode(enc)
            exec_lines.append(cmd)
        if self.startup_command:
            exec_lines.append(self.startup_command)

        # Create the underlying shell class and Engine
        # shell_class = import_item(self.master_config.Global.shell_class)
        # print self.config
        try:
            self.engine = EngineFactory(config=config, log=self.log)
        except:
            self.log.error("Couldn't start the Engine", exc_info=True)
            self.exit(1)

class IPEngineApp(BaseParallelApplication):

    name = 'ipengine'
    description = _description
    examples = _examples
    config_file_name = Unicode(default_config_file_name)
    classes = List([ProfileDir, Session, EngineFactory, Kernel, MPI])

    startup_script = Unicode(u'', config=True,
        help='specify a script to be run at startup')
    startup_command = Unicode('', config=True,
            help='specify a command to be run at startup')

    url_file = Unicode(u'', config=True,
        help="""The full location of the file containing the connection information for
        the controller. If this is not given, the file must be in the
        security directory of the cluster directory.  This location is
        resolved using the `profile` or `profile_dir` options.""",
        )
    wait_for_url_file = Float(5, config=True,
        help="""The maximum number of seconds to wait for url_file to exist.
        This is useful for batch-systems and shared-filesystems where the
        controller and engine are started at the same time and it
        may take a moment for the controller to write the connector files.""")

    url_file_name = Unicode(u'ipcontroller-engine.json', config=True)

    def _cluster_id_changed(self, name, old, new):
        if new:
            base = 'ipcontroller-%s' % new
        else:
            base = 'ipcontroller'
        self.url_file_name = "%s-engine.json" % base

    log_url = Unicode('', config=True,
        help="""The URL for the iploggerapp instance, for forwarding
        logging to a central location.""")

    aliases = Dict(aliases)
    flags = Dict(flags)

    # def find_key_file(self):
    #     """Set the key file.
    # 
    #     Here we don't try to actually see if it exists for is valid as that
    #     is hadled by the connection logic.
    #     """
    #     config = self.master_config
    #     # Find the actual controller key file
    #     if not config.Global.key_file:
    #         try_this = os.path.join(
    #             config.Global.profile_dir, 
    #             config.Global.security_dir,
    #             config.Global.key_file_name
    #         )
    #         config.Global.key_file = try_this
        
    def find_url_file(self):
        """Set the url file.

        Here we don't try to actually see if it exists for is valid as that
        is hadled by the connection logic.
        """
        config = self.config
        # Find the actual controller key file
        if not self.url_file:
            self.url_file = os.path.join(
                self.profile_dir.security_dir,
                self.url_file_name
            )
    
    def load_connector_file(self):
        """load config from a JSON connector file,
        at a *lower* priority than command-line/config files.
        """
        
        self.log.info("Loading url_file %r"%self.url_file)
        config = self.config
        
        with open(self.url_file) as f:
            d = json.loads(f.read())
        
        if 'exec_key' in d:
            config.Session.key = asbytes(d['exec_key'])
        
        try:
            config.EngineFactory.location
        except AttributeError:
            config.EngineFactory.location = d['location']
        
        d['url'] = disambiguate_url(d['url'], config.EngineFactory.location)
        try:
            config.EngineFactory.url
        except AttributeError:
            config.EngineFactory.url = d['url']
        
        try:
            config.EngineFactory.sshserver
        except AttributeError:
            config.EngineFactory.sshserver = d['ssh']
#.........这里部分代码省略.........

class IPEngineApp(BaseParallelApplication):

    name = 'ipengine'
    description = _description
    examples = _examples
    config_file_name = Unicode(default_config_file_name)
    classes = List([ProfileDir, Session, EngineFactory, Kernel, MPI])

    startup_script = Unicode('', config=True,
        help='specify a script to be run at startup')
    startup_command = Unicode('', config=True,
            help='specify a command to be run at startup')

    url_file = Unicode('', config=True,
        help="""The full location of the file containing the connection information for
        the controller. If this is not given, the file must be in the
        security directory of the cluster directory.  This location is
        resolved using the `profile` or `profile_dir` options.""",
        )
    wait_for_url_file = Float(5, config=True,
        help="""The maximum number of seconds to wait for url_file to exist.
        This is useful for batch-systems and shared-filesystems where the
        controller and engine are started at the same time and it
        may take a moment for the controller to write the connector files.""")

    url_file_name = Unicode('ipcontroller-engine.json', config=True)

    def _cluster_id_changed(self, name, old, new):
        if new:
            base = 'ipcontroller-%s' % new
        else:
            base = 'ipcontroller'
        self.url_file_name = "%s-engine.json" % base

    log_url = Unicode('', config=True,
        help="""The URL for the iploggerapp instance, for forwarding
        logging to a central location.""")
    
    # an IPKernelApp instance, used to setup listening for shell frontends
    kernel_app = Instance(IPKernelApp)

    aliases = Dict(aliases)
    flags = Dict(flags)
    
    @property
    def kernel(self):
        """allow access to the Kernel object, so I look like IPKernelApp"""
        return self.engine.kernel

    def find_url_file(self):
        """Set the url file.

        Here we don't try to actually see if it exists for is valid as that
        is hadled by the connection logic.
        """
        config = self.config
        # Find the actual controller key file
        if not self.url_file:
            self.url_file = os.path.join(
                self.profile_dir.security_dir,
                self.url_file_name
            )
    
    def load_connector_file(self):
        """load config from a JSON connector file,
        at a *lower* priority than command-line/config files.
        """
        
        self.log.info("Loading url_file %r", self.url_file)
        config = self.config
        
        with open(self.url_file) as f:
            d = json.loads(f.read())
        
        if 'exec_key' in d:
            config.Session.key = cast_bytes(d['exec_key'])
        
        try:
            config.EngineFactory.location
        except AttributeError:
            config.EngineFactory.location = d['location']
        
        d['url'] = disambiguate_url(d['url'], config.EngineFactory.location)
        try:
            config.EngineFactory.url
        except AttributeError:
            config.EngineFactory.url = d['url']
        
        try:
            config.EngineFactory.sshserver
        except AttributeError:
            config.EngineFactory.sshserver = d['ssh']
    
    def bind_kernel(self, **kwargs):
        """Promote engine to listening kernel, accessible to frontends."""
        if self.kernel_app is not None:
            return
        
        self.log.info("Opening ports for direct connections as an IPython kernel")
        
#.........这里部分代码省略.........

class IPEngineApp(BaseParallelApplication):

    name = Unicode('ipengine')
    description = Unicode(_description)
    config_file_name = Unicode(default_config_file_name)
    classes = List([ProfileDir, Session, EngineFactory, Kernel, MPI])

    startup_script = Unicode('', config=True,
        help='specify a script to be run at startup')
    startup_command = Unicode('', config=True,
            help='specify a command to be run at startup')

    url_file = Unicode('', config=True,
        help="""The full location of the file containing the connection information for
        the controller. If this is not given, the file must be in the
        security directory of the cluster directory.  This location is
        resolved using the `profile` or `profile_dir` options.""",
        )

    url_file_name = Unicode('ipcontroller-engine.json')
    log_url = Unicode('', config=True,
        help="""The URL for the iploggerapp instance, for forwarding
        logging to a central location.""")

    aliases = Dict(dict(
        file = 'IPEngineApp.url_file',
        c = 'IPEngineApp.startup_command',
        s = 'IPEngineApp.startup_script',

        ident = 'Session.session',
        user = 'Session.username',
        exec_key = 'Session.keyfile',

        url = 'EngineFactory.url',
        ip = 'EngineFactory.ip',
        transport = 'EngineFactory.transport',
        port = 'EngineFactory.regport',
        location = 'EngineFactory.location',

        timeout = 'EngineFactory.timeout',

        profile = "IPEngineApp.profile",
        profile_dir = 'ProfileDir.location',

        mpi = 'MPI.use',

        log_level = 'IPEngineApp.log_level',
        log_url = 'IPEngineApp.log_url'
    ))

    # def find_key_file(self):
    #     """Set the key file.
    # 
    #     Here we don't try to actually see if it exists for is valid as that
    #     is hadled by the connection logic.
    #     """
    #     config = self.master_config
    #     # Find the actual controller key file
    #     if not config.Global.key_file:
    #         try_this = os.path.join(
    #             config.Global.profile_dir, 
    #             config.Global.security_dir,
    #             config.Global.key_file_name
    #         )
    #         config.Global.key_file = try_this
        
    def find_url_file(self):
        """Set the key file.

        Here we don't try to actually see if it exists for is valid as that
        is hadled by the connection logic.
        """
        config = self.config
        # Find the actual controller key file
        if not self.url_file:
            self.url_file = os.path.join(
                self.profile_dir.security_dir,
                self.url_file_name
            )
    def init_engine(self):
        # This is the working dir by now.
        sys.path.insert(0, '')
        config = self.config
        # print config
        self.find_url_file()

        # if os.path.exists(config.Global.key_file) and config.Global.secure:
        #     config.SessionFactory.exec_key = config.Global.key_file
        if os.path.exists(self.url_file):
            with open(self.url_file) as f:
                d = json.loads(f.read())
                for k,v in d.items():
                    if isinstance(v, str):
                        d[k] = v.encode()
            if d['exec_key']:
                config.Session.key = d['exec_key']
            d['url'] = disambiguate_url(d['url'], d['location'])
            config.EngineFactory.url = d['url']
            config.EngineFactory.location = d['location']
        
#.........这里部分代码省略.........

class IPEngineApp(ApplicationWithClusterDir):

    name = u'ipengine'
    description = _description
    command_line_loader = IPEngineAppConfigLoader
    default_config_file_name = default_config_file_name
    auto_create_cluster_dir = True

    def create_default_config(self):
        super(IPEngineApp, self).create_default_config()

        # The engine should not clean logs as we don't want to remove the
        # active log files of other running engines.
        self.default_config.Global.clean_logs = False
        self.default_config.Global.secure = True

        # Global config attributes
        self.default_config.Global.exec_lines = []
        self.default_config.Global.extra_exec_lines = ''
        self.default_config.Global.extra_exec_file = u''

        # Configuration related to the controller
        # This must match the filename (path not included) that the controller
        # used for the FURL file.
        self.default_config.Global.url_file = u''
        self.default_config.Global.url_file_name = u'ipcontroller-engine.json'
        # If given, this is the actual location of the controller's FURL file.
        # If not, this is computed using the profile, app_dir and furl_file_name
        # self.default_config.Global.key_file_name = u'exec_key.key'
        # self.default_config.Global.key_file = u''

        # MPI related config attributes
        self.default_config.MPI.use = ''
        self.default_config.MPI.mpi4py = mpi4py_init
        self.default_config.MPI.pytrilinos = pytrilinos_init

    def post_load_command_line_config(self):
        pass

    def pre_construct(self):
        super(IPEngineApp, self).pre_construct()
        # self.find_cont_url_file()
        self.find_url_file()
        if self.master_config.Global.extra_exec_lines:
            self.master_config.Global.exec_lines.append(self.master_config.Global.extra_exec_lines)
        if self.master_config.Global.extra_exec_file:
            enc = sys.getfilesystemencoding() or 'utf8'
            cmd="execfile(%r)"%self.master_config.Global.extra_exec_file.encode(enc)
            self.master_config.Global.exec_lines.append(cmd)

    # def find_key_file(self):
    #     """Set the key file.
    # 
    #     Here we don't try to actually see if it exists for is valid as that
    #     is hadled by the connection logic.
    #     """
    #     config = self.master_config
    #     # Find the actual controller key file
    #     if not config.Global.key_file:
    #         try_this = os.path.join(
    #             config.Global.cluster_dir, 
    #             config.Global.security_dir,
    #             config.Global.key_file_name
    #         )
    #         config.Global.key_file = try_this
        
    def find_url_file(self):
        """Set the key file.

        Here we don't try to actually see if it exists for is valid as that
        is hadled by the connection logic.
        """
        config = self.master_config
        # Find the actual controller key file
        if not config.Global.url_file:
            try_this = os.path.join(
                config.Global.cluster_dir, 
                config.Global.security_dir,
                config.Global.url_file_name
            )
            config.Global.url_file = try_this
        
    def construct(self):
        # This is the working dir by now.
        sys.path.insert(0, '')
        config = self.master_config
        # if os.path.exists(config.Global.key_file) and config.Global.secure:
        #     config.SessionFactory.exec_key = config.Global.key_file
        if os.path.exists(config.Global.url_file):
            with open(config.Global.url_file) as f:
                d = json.loads(f.read())
                for k,v in d.iteritems():
                    if isinstance(v, unicode):
                        d[k] = v.encode()
            if d['exec_key']:
                config.SessionFactory.exec_key = d['exec_key']
            d['url'] = disambiguate_url(d['url'], d['location'])
            config.RegistrationFactory.url=d['url']
            config.EngineFactory.location = d['location']
        
#.........这里部分代码省略.........

    def init_engine(self):
        # This is the working dir by now.
        sys.path.insert(0, "")
        config = self.config
        # print config
        self.find_url_file()

        # was the url manually specified?
        keys = set(self.config.EngineFactory.keys())
        keys = keys.union(set(self.config.RegistrationFactory.keys()))

        if keys.intersection(set(["ip", "url", "port"])):
            # Connection info was specified, don't wait for the file
            url_specified = True
            self.wait_for_url_file = 0
        else:
            url_specified = False

        if self.wait_for_url_file and not os.path.exists(self.url_file):
            self.log.warn("url_file %r not found" % self.url_file)
            self.log.warn("Waiting up to %.1f seconds for it to arrive." % self.wait_for_url_file)
            tic = time.time()
            while not os.path.exists(self.url_file) and (time.time() - tic < self.wait_for_url_file):
                # wait for url_file to exist, for up to 10 seconds
                time.sleep(0.1)

        if os.path.exists(self.url_file):
            self.log.info("Loading url_file %r" % self.url_file)
            with open(self.url_file) as f:
                d = json.loads(f.read())
                for k, v in d.items():
                    if isinstance(v, str):
                        d[k] = v.encode()
            if d["exec_key"]:
                config.Session.key = d["exec_key"]
            d["url"] = disambiguate_url(d["url"], d["location"])
            config.EngineFactory.url = d["url"]
            config.EngineFactory.location = d["location"]
        elif not url_specified:
            self.log.critical("Fatal: url file never arrived: %s" % self.url_file)
            self.exit(1)

        try:
            exec_lines = config.Kernel.exec_lines
        except AttributeError:
            config.Kernel.exec_lines = []
            exec_lines = config.Kernel.exec_lines

        if self.startup_script:
            enc = sys.getfilesystemencoding() or "utf8"
            cmd = "execfile(%r)" % self.startup_script.encode(enc)
            exec_lines.append(cmd)
        if self.startup_command:
            exec_lines.append(self.startup_command)

        # Create the underlying shell class and Engine
        # shell_class = import_item(self.master_config.Global.shell_class)
        # print self.config
        try:
            self.engine = EngineFactory(config=config, log=self.log)
        except:
            self.log.error("Couldn't start the Engine", exc_info=True)
            self.exit(1)