Python containers.dict_merge函数代码示例

    def _load_capabilities(self):
        self._cap_initialized = []  # List of capability constants initialized in container
        self._capabilities = []     # List of capability constants active in container
        self._cap_instances = {}    # Dict mapping capability->manager instance

        self._cap_definitions = Config(["res/config/container_capabilities.yml"]).data['capabilities']

        profile_filename = CFG.get_safe("container.profile", "development")
        if not profile_filename.endswith(".yml"):
            profile_filename = "res/profile/%s.yml" % profile_filename
        log.info("Loading CC capability profile from file: %s", profile_filename)
        profile_cfg = Config([profile_filename]).data
        if not isinstance(profile_cfg, dict) or profile_cfg['type'] != "profile" or not "profile" in profile_cfg:
            raise ContainerError("Container capability profile invalid: %s" % profile_filename)

        self.cap_profile = profile_cfg['profile']

        if "capabilities" in self.cap_profile and self.cap_profile['capabilities']:
            dict_merge(self._cap_definitions, self.cap_profile['capabilities'], True)

        CCAP.clear()
        cap_list = self._cap_definitions.keys()
        CCAP.update(zip(cap_list, cap_list))

        if "config" in self.cap_profile and self.cap_profile['config']:
            log.info("Container CFG was changed based on profile: %s", profile_filename)

    def start_app(self, appdef=None, config=None):
        """
        @brief Start an app from an app definition.
        Note: apps can come in one of 2 variants:
        1 processapp: In-line defined process to be started
        2 regular app: Full app definition
        """
        log.debug("AppManager.start_app(appdef=%s) ..." % appdef)

        appdef = DotDict(appdef)

        if 'config' in appdef:
            app_cfg = appdef.config.copy()
            if config:
                dict_merge(app_cfg, config, inplace=True)
            config = app_cfg

        if 'processapp' in appdef:
            # Case 1: Appdef contains definition of process to start
            name, module, cls = appdef.processapp
            try:
                pid = self.container.spawn_process(name, module, cls, config)
                appdef._pid = pid
                self.apps.append(appdef)
            except Exception, ex:
                log.exception("Appl %s start from processapp failed" % appdef.name)

    def _get_execution_engine_config(self):
        ee_base_cfg = CFG.get_safe("container.execution_engine") or {}
        if ee_base_cfg.get("type", None) != "scioncc":
            raise ContainerConfigError("Execution engine config invalid: %s", ee_base_cfg)

        ee_cfg = deepcopy(ee_base_cfg)

        # If we are a child process, merge in child config override
        proc_name = multiprocessing.current_process().name
        ee_cfg["container"] = dict(child_proc_name=proc_name, is_child=False)
        child_cfgs = ee_base_cfg.get("child_configs", None) or {}
        if proc_name.startswith("Container-child-"):
            ee_cfg["container"]["is_child"] = True
            if proc_name in child_cfgs:
                log.info("Applying execution engine config override for child: %s", proc_name)
                dict_merge(ee_cfg, child_cfgs[proc_name], inplace=True)
            else:
                for cfg_name, ch_cfg in child_cfgs.iteritems():
                    pattern = ch_cfg.get("name_pattern", None)
                    if pattern and re.match(pattern, proc_name):
                        log.info("Applying execution engine config override %s for child: %s", cfg_name, proc_name)
                        dict_merge(ee_cfg, ch_cfg, inplace=True)
                        break

        ee_cfg.pop("child_configs", None)
        return ee_cfg

    def __init__(self, *args, **kwargs):
        BaseContainerAgent.__init__(self, *args, **kwargs)

        # set id and name (as they are set in base class call)
        self.id = string.replace('%s_%d' % (os.uname()[1], os.getpid()), ".", "_")
        self.name = "cc_agent_%s" % self.id

        Container.instance = self

        # TODO: Bug: Replacing CFG instance not work because references are already public. Update directly
        dict_merge(CFG, kwargs)
        from pyon.core import bootstrap
        bootstrap.sys_name = CFG.system.name or bootstrap.sys_name
        log.debug("Container (sysname=%s) initializing ..." % bootstrap.sys_name)

        # Keep track of the overrides from the command-line, so they can trump app/rel file data
        self.spawn_args = DictModifier(CFG, kwargs)

        # Load object and service registry etc.
        bootstrap_pyon()

        # Create this Container's specific ExchangeManager instance
        self.ex_manager = ExchangeManager(self)

        # Create this Container's specific ProcManager instance
        self.proc_manager = ProcManager(self)

        # Create this Container's specific AppManager instance
        self.app_manager = AppManager(self)

        # DatastoreManager - controls access to Datastores (both mock and couch backed)
        self.datastore_manager = DatastoreManager()
        
        log.debug("Container initialized, OK.")

    def start_rel(self, rel=None, config=None):
        """
        @brief Recurse over the rel and start apps defined there.
        Note: apps in a rel file can come in one of 2 forms:
        1 processapp: In-line defined process to be started as app
        2 app file: Reference to an app definition in an app file
        If the rel file provides an app config block, it is provided to spawn the process.
        Any given function config dict is merged on top of this.
        """
        log.debug("AppManager.start_rel(rel=%s) ...", rel)

        if rel is None:
            return

        if self.use_pd:
            log.info("Sending rel file to PD")
            import json
            rel_def = json.loads(json.dumps(rel))     # HACK to get rid of OrderedDict (not serializable)
            cmd_res = self.pd_client.start_rel_blocking(rel_def, timeout=None)
            return cmd_res

        for rel_app_cfg in rel.apps:
            name = rel_app_cfg.name
            log.debug("app definition in rel: %s" % str(rel_app_cfg))

            if 'processapp' in rel_app_cfg:
                # Case 1: Rel contains definition of process to start as app
                name, module, cls = rel_app_cfg.processapp

                rel_cfg = None
                if 'config' in rel_app_cfg:
                    rel_cfg = deepcopy(rel_app_cfg.config)
                    if config:
                        dict_merge(rel_cfg, config, inplace=True)

                if 'replicas' in rel_app_cfg:
                    proc_replicas = int(rel_app_cfg["replicas"])
                    if self.max_proc_replicas > 0:
                        if proc_replicas > self.max_proc_replicas:
                            log.info("Limiting number of proc replicas to %s from %s", self.max_proc_replicas, proc_replicas)
                        proc_replicas = min(proc_replicas, self.max_proc_replicas)
                    if proc_replicas < 1 or proc_replicas > 100:
                        log.warn("Invalid number of process replicas: %s", proc_replicas)
                        proc_replicas = 1
                    for i in xrange(proc_replicas):
                        proc_name = "%s.%s" % (name, i) if i else name
                        self.container.spawn_process(proc_name, module, cls, rel_cfg)
                else:
                    self.container.spawn_process(name, module, cls, rel_cfg)
                self.apps.append(DotDict(type="application", name=name, processapp=rel_app_cfg.processapp))

            else:
                # Case 2: Rel contains reference to app file to start
                app_file_path = 'res/apps/%s.yml' % (name)
                rel_cfg = rel_app_cfg.get('config', None)
                if config:
                    dict_merge(rel_cfg, config, inplace=True)
                self.start_app_from_url(app_file_path, config=rel_cfg)

    def __init__(self, *args, **kwargs):
        BaseContainerAgent.__init__(self, *args, **kwargs)

        self._is_started = False

        # set id and name (as they are set in base class call)
        self.id = string.replace('%s_%d' % (os.uname()[1], os.getpid()), ".", "_")
        self.name = "cc_agent_%s" % self.id

        Container.instance = self

        # TODO: Bug: Replacing CFG instance not work because references are already public. Update directly
        dict_merge(CFG, kwargs, inplace=True)
        from pyon.core import bootstrap
        bootstrap.container_instance = self
        bootstrap.assert_configuration(CFG)
        log.debug("Container (sysname=%s) initializing ..." % bootstrap.get_sys_name())

        # Keep track of the overrides from the command-line, so they can trump app/rel file data
        self.spawn_args = kwargs

        # Load object and service registry etc.
        bootstrap_pyon()

        # Create this Container's specific ExchangeManager instance
        self.ex_manager = ExchangeManager(self)

        # Create this Container's specific ProcManager instance
        self.proc_manager = ProcManager(self)

        # Create this Container's specific AppManager instance
        self.app_manager = AppManager(self)

        # DatastoreManager - controls access to Datastores (both mock and couch backed)
        self.datastore_manager = DatastoreManager()

        # File System - Interface to the OS File System, using correct path names and setups
        self.file_system = FileSystem(CFG)

        # Governance Controller - manages the governance related interceptors
        self.governance_controller = GovernanceController(self)

        # sFlow manager - controls sFlow stat emission
        self.sflow_manager = SFlowManager(self)

        # Coordinates the container start
        self._is_started = False
        self._capabilities = []
        self._status = "INIT"

        # protection for when the container itself is used as a Process for clients
        self.container = self

        log.debug("Container initialized, OK.")

def apply_profile_configuration(system_cfg, bootstrap_config):
    profile_filename = bootstrap_config.get_safe("container.profile", None)
    if not profile_filename:
        return
    if not profile_filename.endswith(".yml"):
        profile_filename = "res/profile/%s.yml" % profile_filename
    from pyon.util.config import Config
    profile_cfg = Config([profile_filename]).data
    config_override = profile_cfg.get_safe("profile.config")
    if config_override and isinstance(config_override, dict):
        from pyon.util.containers import dict_merge
        dict_merge(system_cfg, config_override, inplace=True)

    def _generate_driver_config(self):
        log.debug("_generate_driver_config for %s", self.agent_instance_obj.name)
        # get default config
        driver_config = super(ExternalDatasetAgentConfigurationBuilder, self)._generate_driver_config()

        agent_instance_obj = self.agent_instance_obj
        agent_obj = self._get_agent()

        parser_cfg = copy.deepcopy(agent_obj.parser_default_config)
        poller_cfg = copy.deepcopy(agent_obj.poller_default_config)

        # Create driver config.
        base_driver_config = {
            'parser': {
                'uri': agent_obj.parser_uri,
                'module': agent_obj.parser_module,
                'class': agent_obj.parser_class,
                'config': parser_cfg,
            },
            'poller': {
                'uri': agent_obj.poller_uri,
                'module': agent_obj.poller_module,
                'class': agent_obj.poller_class,
                'config': poller_cfg,
            },
        }

        res_driver_config = dict_merge(base_driver_config, driver_config)

        return res_driver_config

    def _generate_skeleton_config_block(self):
        log.info("Generating skeleton config block for %s", self.agent_instance_obj.name)

        # merge the agent config into the default config
        agent_config = dict_merge(self._get_agent().agent_default_config, self.agent_instance_obj.agent_config, True)

        org_obj = self._generate_org()

        # Create agent_config.
        agent_config['instance_id']        = self.agent_instance_obj._id
        agent_config['instance_name']        = self.agent_instance_obj.name
        agent_config['org_governance_name']  = org_obj.org_governance_name if org_obj else ''
        agent_config['provider_id']          = org_obj._id if org_obj else ''
        agent_config['actor_id']             = self.actor_id
        agent_config['device_type']          = self._generate_device_type()
        agent_config['driver_config']        = self._generate_driver_config()
        agent_config['stream_config']        = self._generate_stream_config()
        agent_config['agent']                = self._generate_agent_config()
        agent_config['aparam_alerts_config'] = self._generate_alerts_config()
        agent_config['startup_config']       = self._generate_startup_config()
        agent_config['children']             = self._generate_children()

        log.info("DONE generating skeleton config block for %s", self.agent_instance_obj.name)

        return agent_config

    def _call_plugins(self, method, process, config, **kwargs):
        bootstrap_plugins = config.get_safe("bootstrap_plugins", None)
        if bootstrap_plugins is None:
            log.warn("Bootstrapper called without bootstrap_plugins config")

        # Finding the system actor ID. If found, construct call context headers.
        # This may be called very early in bootstrap with no system actor yet existing
        system_actor, _ = process.container.resource_registry.find_resources(
            RT.ActorIdentity, name=self.CFG.system.system_actor, id_only=True
        )
        system_actor_id = system_actor[0] if system_actor else "anonymous"

        actor_headers = {
            "ion-actor-id": system_actor_id,
            "ion-actor-roles": {"ION": ["ION_MANAGER", "ORG_MANAGER"]} if system_actor else {},
        }

        # Set the call context of the current process
        with process.push_context(actor_headers):

            for plugin_info in bootstrap_plugins:
                plugin_mod, plugin_cls = plugin_info.get("plugin", [None, None])
                plugin_cfg = plugin_info.get("config", None)
                plugin_cfg = dict_merge(config, plugin_cfg) if plugin_cfg is not None else config

                try:
                    log.info("Bootstrapping plugin %s.%s ...", plugin_mod, plugin_cls)
                    plugin = for_name(plugin_mod, plugin_cls)
                    plugin_func = getattr(plugin, method)
                    plugin_func(process, plugin_cfg, **kwargs)
                except AbortBootstrap as abort:
                    raise
                except Exception as ex:
                    log.exception("Error bootstrapping plugin %s.%s", plugin_mod, plugin_cls)

    def _call_plugins(self, method, process, config, **kwargs):
        bootstrap_plugins = config.get_safe("bootstrap_plugins", None)
        if bootstrap_plugins is None:
            log.warn("Bootstrapper called without bootstrap_plugins config")

        # Finding the system actor ID. If found, construct call context headers.
        # This may be called very early in bootstrap with no system actor yet existing
        system_actor = get_system_actor()
        if system_actor:
            actor_headers = get_system_actor_header(system_actor)
        else:
            # Use default actor headers, not roles.
            actor_headers = build_actor_header()

        # Set the call context of the current process
        with process.push_context(actor_headers):

            for plugin_info in bootstrap_plugins:
                plugin_mod, plugin_cls = plugin_info.get("plugin", [None,None])
                plugin_cfg = plugin_info.get("config", None)
                plugin_cfg = dict_merge(config, plugin_cfg) if plugin_cfg is not None else config

                try:
                    log.info("Bootstrapping plugin %s.%s ...", plugin_mod, plugin_cls)
                    plugin = for_name(plugin_mod, plugin_cls)
                    plugin_func = getattr(plugin, method)
                    plugin_func(process, plugin_cfg, **kwargs)
                except AbortBootstrap as abort:
                    raise
                except Exception as ex:
                    log.exception("Error bootstrapping plugin %s.%s", plugin_mod, plugin_cls)

    def do_work(container):
        """
        Performs initial startup actions with the container as requested in arguments.
        Then remains in container shell or infinite wait until container stops.
        Returns when container should stop. Raises an exception if anything failed.
        """
        if opts.proc:
            # Run a one-off process (with the -x argument)
            mod, proc = opts.proc.rsplit('.', 1)
            print "pycc: Starting process %s" % opts.proc
            container.spawn_process(proc, mod, proc, config={'process':{'type':'immediate'}})
            # And end
            return

        if opts.rel:
            # Start a rel file
            start_ok = container.start_rel_from_url(opts.rel)
            if not start_ok:
                raise Exception("Cannot start deploy file '%s'" % opts.rel)

        if opts.mx:
            container.spawn_process("ContainerUI", "ion.core.containerui", "ContainerUI")
            print "pycc: Container UI started ... listening on http://localhost:8080"

        if opts.signalparent:
            import os
            import signal
            print 'pycc: Signal parent pid %d that pycc pid %d service start process is complete...' % (os.getppid(), os.getpid())
            os.kill(os.getppid(), signal.SIGUSR1)

            def is_parent_gone():
                while os.getppid() != 1:
                    gevent.sleep(1)
                print 'pycc: Now I am an orphan ... notifying serve_forever to stop'
                os.kill(os.getpid(), signal.SIGINT)
            import gevent
            ipg = gevent.spawn(is_parent_gone)
            from pyon.util.containers import dict_merge
            from pyon.public import CFG
            dict_merge(CFG, {'system':{'watch_parent': ipg}}, True)
        if not opts.noshell and not opts.daemon:
            # Keep container running while there is an interactive shell
            from pyon.container.shell_api import get_shell_api
            setup_ipython(get_shell_api(container))
        else:
            # Keep container running until process terminated
            container.serve_forever()

 def acquire_data(self, streaming_args=None):
     if self.current_state != self.AGENTSTATE_CONNECTED:
         raise BadRequest("Illegal agent state: %s" % self.current_state)
     try:
         args = dict_merge(self.agent_config, streaming_args) if streaming_args else self.agent_config
         res = self.on_acquire_data(args)
     except Exception:
         self.current_state = self.AGENTSTATE_ERROR
         raise

    def test_dict_merge(self):
        # dict_merge(base, upd, inplace=False):
        org_dict = {"a":"str_a", "d": {"d-x": 1, "d-y": None, "d-d": {"d-d-1": 1, "d-d-2": 2}}}

        base_dict = copy.deepcopy(org_dict)
        dd = DictDiffer(base_dict, org_dict)
        self.assertTrue(len(base_dict), 2)
        self.assertTrue(len(dd.unchanged()), len(base_dict))

        # Case 1: Add new value
        delta_dict = {"c" : "NEW_C"}
        mod_dict = dict_merge(base_dict, delta_dict)
        dd = DictDiffer(base_dict, org_dict)
        self.assertTrue(len(base_dict), 2)
        self.assertTrue(len(dd.unchanged()), len(base_dict))

        dd = DictDiffer(mod_dict, org_dict)
        self.assertTrue(len(mod_dict), 3)
        self.assertTrue(len(dd.unchanged()), len(org_dict))
        self.assertTrue(dd.added(), 1)

        # Case 2: Change simple type value
        delta_dict = {"a" : 5}
        base_dict = copy.deepcopy(org_dict)
        mod_dict = dict_merge(base_dict, delta_dict)

        dd = DictDiffer(mod_dict, org_dict)
        self.assertTrue(len(mod_dict), len(org_dict))
        self.assertTrue(len(dd.unchanged()), len(org_dict)-1)
        self.assertTrue(len(dd.changed()), 1)
        self.assertTrue(mod_dict['a'], 5)

        # Case 3: Add new value on lower level
        delta_dict = {"d": {"new":"NEW_ENTRY"}}
        base_dict = copy.deepcopy(org_dict)
        mod_dict = dict_merge(base_dict, delta_dict)

        dd = DictDiffer(mod_dict, org_dict)
        self.assertTrue(len(mod_dict), len(org_dict))
        self.assertTrue(len(mod_dict['d']), len(org_dict['d']) + 1)
        self.assertTrue(mod_dict['d']['new'], "NEW_ENTRY")
        dd = DictDiffer(mod_dict['d'], org_dict['d'])
        self.assertTrue(len(dd.unchanged()), len(org_dict['d']))
        self.assertTrue(dd.added(), 1)

    def start_app(self, appdef=None, config=None):
        """
        @brief Start an app from an app definition.
        Note: apps can come in one of 2 variants:
        1 processapp: In-line defined process to be started
        2 regular app: Full app definition
        """
        log.debug("AppManager.start_app(appdef=%s) ..." % appdef)

        appdef = DotDict(appdef)

        if 'config' in appdef:
            app_cfg = deepcopy(appdef.config)
            if config:
                dict_merge(app_cfg, config, inplace=True)
            config = app_cfg

        if 'processapp' in appdef:
            # Case 1: Appdef contains definition of process to start
            name, module, cls = appdef.processapp
            try:
                pid = self.container.spawn_process(name, module, cls, config)
                appdef._pid = pid
                self.apps.append(appdef)
            except Exception:
                log.exception("Appl %s start from processapp failed" % appdef.name)
        else:
            # Case 2: Appdef contains full app start params
            modpath = appdef.mod
            try:
                mod = named_any(modpath)
                appdef._mod_loaded = mod

                # Start the app
                supid, state = mod.start(self.container, START_PERMANENT, appdef, config)
                appdef._supid = supid
                appdef._state = state

                log.debug("App '%s' started. Root sup-id=%s" % (appdef.name, supid))

                self.apps.append(appdef)
            except Exception:
                log.exception("Appl %s start from appdef failed" % appdef.name)