Python EnhancedResourceRegistryClient.cache_predicate方法代码示例

# 需要导入模块: from ion.util.enhanced_resource_registry_client import EnhancedResourceRegistryClient [as 别名]
# 或者: from ion.util.enhanced_resource_registry_client.EnhancedResourceRegistryClient import cache_predicate [as 别名]
class AgentConfigurationBuilder(object):

    def __init__(self, clients, RR2=None):
        self.clients = clients
        self.RR2 = RR2

        if self.RR2 is None:
            log.warn("Creating new RR2")
            self.RR2 = EnhancedResourceRegistryClient(self.clients.resource_registry)

        if not isinstance(self.RR2, EnhancedResourceRegistryClient):
            raise AssertionError("Type of self.RR2 is %s not %s" %
                                 (type(self.RR2), type(EnhancedResourceRegistryClient)))

        self.agent_instance_obj = None
        self.associated_objects = None
        self.last_id            = None
        self.will_launch        = False
        self.generated_config   = False

    def _predicates_to_cache(self):
        return [PRED.hasOutputProduct,
                #PRED.hasStream,
                #PRED.hasStreamDefinition,
                PRED.hasAgentInstance,
                PRED.hasAgentDefinition,
                PRED.hasDataset,
                PRED.hasDevice,
                PRED.hasNetworkParent,
                #PRED.hasParameterContext,
                PRED.hasDeployment,
                ]

    def _resources_to_cache(self):
        return [#RT.StreamDefinition,
                RT.ParameterDictionary,
                #RT.ParameterContext,
                RT.Deployment,
                ]

    def _update_cached_predicates(self):
        # cache some predicates for in-memory lookups
        preds = self._predicates_to_cache()
        log.debug("updating cached predicates: %s" % preds)
        time_caching_start = get_ion_ts()
        for pred in preds:
            log.debug(" - %s", pred)
            self.RR2.cache_predicate(pred)
        time_caching_stop = get_ion_ts()

        total_time = int(time_caching_stop) - int(time_caching_start)

        log.info("Cached %s predicates in %s seconds", len(preds), total_time / 1000.0)

    def _update_cached_resources(self):
        # cache some resources for in-memory lookups
        rsrcs = self._resources_to_cache()
        log.debug("updating cached resources: %s" % rsrcs)
        time_caching_start = get_ion_ts()
        for r in rsrcs:
            log.debug(" - %s", r)
            self.RR2.cache_resources(r)
        time_caching_stop = get_ion_ts()

        total_time = int(time_caching_stop) - int(time_caching_start)

        log.info("Cached %s resource types in %s seconds", len(rsrcs), total_time / 1000.0)

    def _clear_caches(self):
        log.warn("Clearing caches")
        for r in self._resources_to_cache():
            self.RR2.clear_cached_resource(r)

        for p in self._predicates_to_cache():
            self.RR2.clear_cached_predicate(p)
            

    def _lookup_means(self):
        """
        return a dict indicating how various related resources will be looked up

        The dict is keyed on association type:
        PRED.hasAgentInstance -> device type
        PRED.hasModel -> model type
        PRED.hasAgentDefinition -> agent type
        """
        raise NotImplementedError("Extender of class must implement this")

    def _augment_dict(self, title, basedict, newitems):
        # TODO: pyon.util.containers has dict_merge for this purpose (without logs)
        for k, v in newitems.iteritems():
            if k in basedict:
                prev_v = basedict[k]
                # just warn if the new value is different
                if v != prev_v:
                    log.warn("Overwriting %s[%s] of '%s' with '%s'", title, k, prev_v, v)
                else:
                    log.debug("Overwriting %s[%s] with same value already assigned '%s'",
                              title, k, v)
            basedict[k] = v
#.........这里部分代码省略.........

# 需要导入模块: from ion.util.enhanced_resource_registry_client import EnhancedResourceRegistryClient [as 别名]
# 或者: from ion.util.enhanced_resource_registry_client.EnhancedResourceRegistryClient import cache_predicate [as 别名]
        def freeze():

            if isinstance(resource_registry_client, EnhancedResourceRegistryClient):
                RR2 = resource_registry_client
            else:
                RR2 = EnhancedResourceRegistryClient(resource_registry_client)

            for p in predicate_list:
                if not RR2.has_cached_prediate(p):
                    RR2.cache_predicate(p)


            def get_related_resources_partial_fn(predicate_dictionary, resource_whitelist):
                """
                This function generates a resource crawler from 2 data structures representing desired crawl behavior

                The predicate dictionary is keyed on a predicate type, whose value is a 2-tuple of booleans
                  the first boolean is whether to crawl subject-object, the second boolean for object-subject
                  For example: dict([(PRED.hasModel, (False, True)]) would generate a crawler that could find
                               platforms or instruments with a given model

                The resource whitelist is a list of resource types that will be crawled.

                The return value of this function is a function that accepts a resource id and returns a list
                  of associations related (based on crawl behavior)
                """
                log.trace("get_related_resources_partial_fn predicate_dict=%s rsrc_whitelist=%s",
                          predicate_dictionary,
                          resource_whitelist)

                # assertions on data types
                assert type({}) == type(predicate_dictionary)
                for v in predicate_dictionary.values():
                    assert type((True, True)) == type(v)
                assert type([]) == type(resource_whitelist)

                for rt in resource_whitelist:
                    RR2.cache_resources(rt)

                def lookup_fn(resource_id):
                    """
                    return a dict of related resources as dictated by the pred dict and whitelist
                     - the key is the next resource id to crawl
                     - the value is the entire association
                    """
                    retval = {}

                    sto_match = lambda assn: assn.s == resource_id and assn.ot in resource_whitelist
                    ots_match = lambda assn: assn.o == resource_id and assn.st in resource_whitelist
                    for p, (search_sto, search_ots) in predicate_dictionary.iteritems():
                        if search_sto:
                            for a in RR2.filter_cached_associations(p, sto_match):
                                log.trace("lookup_fn matched %s object", a.ot)
                                retval[a.o] = a
                        if search_ots:
                            for a in RR2.filter_cached_associations(p, ots_match):
                                log.trace("lookup_fn matched %s subject", a.st)
                                retval[a.s] = a


                    return retval


                def get_related_resources_h(accum, input_resource_id, recursion_limit):
                    """
                    This is a recursive helper function that does the work of crawling for related resources

                    The accum is a tuple: (set of associations that are deemed "Related", set of "seen" resources)

                    The input resource id is the current resource being crawled

                    The recursion limit decrements with each recursive call, ending at 0.  So -1 for infinity.

                    The return value is a list of associations
                    """
                    if 0 == recursion_limit:
                        return accum

                    if -1000 > recursion_limit:
                        log.warn("Terminating related resource recursion, hit -1000")
                        return accum

                    acc, seen = accum

                    matches = lookup_fn(input_resource_id)
                    log.trace("get_related_resources_h got matches %s",
                              [dict((k, "%s %s %s" % (a.st, a.p, a.ot)) for k, a in matches.iteritems())])

                    unseen = set(matches.keys()) - seen
                    seen.add(input_resource_id)
                    acc  = acc  | set(matches.values())

                    #if log.isEnabledFor(logging.TRACE):
                    #    summary = {}
                    #    for a in acc:
                    #        label = "%s %s %s" % (a.st, a.p, a.ot)
                    #        if not label in summary: summary[label] = 0
                    #        summary[label] += 1
                    #    log.trace("acc2 is now %s", ["%s x%d" % (k, v) for k, v in summary.iteritems()])

#.........这里部分代码省略.........

# 需要导入模块: from ion.util.enhanced_resource_registry_client import EnhancedResourceRegistryClient [as 别名]
# 或者: from ion.util.enhanced_resource_registry_client.EnhancedResourceRegistryClient import cache_predicate [as 别名]
class AgentConfigurationBuilder(object):

    def __init__(self, clients, RR2=None):
        self.clients = clients
        self.RR2 = RR2

        if self.RR2 is None:
            log.warn("Creating new RR2")
            self.RR2 = EnhancedResourceRegistryClient(self.clients.resource_registry)

        if not isinstance(self.RR2, EnhancedResourceRegistryClient):
            raise AssertionError("Type of self.RR2 is %s not %s" %
                                 (type(self.RR2), type(EnhancedResourceRegistryClient)))

        self.agent_instance_obj = None
        self.associated_objects = None
        self.last_id            = None
        self.will_launch        = False
        self.generated_config   = False

    def _predicates_to_cache(self):
        return [PRED.hasOutputProduct,
                #PRED.hasStream,
                #PRED.hasStreamDefinition,
                PRED.hasAgentInstance,
                PRED.hasAgentDefinition,
                PRED.hasDataset,
                PRED.hasDevice,
                PRED.hasNetworkParent,
                #PRED.hasParameterContext,
                ]

    def _resources_to_cache(self):
        return [#RT.StreamDefinition,
                RT.ParameterDictionary,
                #RT.ParameterContext,
                ]

    def _update_cached_predicates(self):
        # cache some predicates for in-memory lookups
        preds = self._predicates_to_cache()
        log.debug("updating cached predicates: %s" % preds)
        time_caching_start = get_ion_ts()
        for pred in preds:
            log.debug(" - %s", pred)
            self.RR2.cache_predicate(pred)
        time_caching_stop = get_ion_ts()

        total_time = int(time_caching_stop) - int(time_caching_start)

        log.info("Cached %s predicates in %s seconds", len(preds), total_time / 1000.0)

    def _update_cached_resources(self):
        # cache some resources for in-memory lookups
        rsrcs = self._resources_to_cache()
        log.debug("updating cached resources: %s" % rsrcs)
        time_caching_start = get_ion_ts()
        for r in rsrcs:
            log.debug(" - %s", r)
            self.RR2.cache_resources(r)
        time_caching_stop = get_ion_ts()

        total_time = int(time_caching_stop) - int(time_caching_start)

        log.info("Cached %s resource types in %s seconds", len(rsrcs), total_time / 1000.0)

    def _clear_caches(self):
        log.warn("Clearing caches")
        for r in self._resources_to_cache():
            self.RR2.clear_cached_resource(r)

        for p in self._predicates_to_cache():
            self.RR2.clear_cached_predicate(p)
            

    def _lookup_means(self):
        """
        return a dict indicating how various related resources will be looked up

        The dict is keyed on association type:
        PRED.hasAgentInstance -> device type
        PRED.hasModel -> model type
        PRED.hasAgentDefinition -> agent type
        """
        raise NotImplementedError("Extender of class must implement this")

    def _augment_dict(self, title, basedict, newitems):
        for k, v in newitems.iteritems():
            if k in basedict:
                prev_v = basedict[k]
                # just warn if the new value is different
                if v != prev_v:
                    log.warn("Overwriting %s[%s] of '%s' with '%s'", title, k, prev_v, v)
                else:
                    log.debug("Overwriting %s[%s] with same value already assigned '%s'",
                              title, k, v)
            basedict[k] = v

    def _check_associations(self):
        assert self.agent_instance_obj
#.........这里部分代码省略.........

# 需要导入模块: from ion.util.enhanced_resource_registry_client import EnhancedResourceRegistryClient [as 别名]
# 或者: from ion.util.enhanced_resource_registry_client.EnhancedResourceRegistryClient import cache_predicate [as 别名]

#.........这里部分代码省略.........
            self.rr.find_objects.reset_mock()
            self.rr.get_association.reset_mock()

        rst()
        self.rr.find_objects.return_value = ([], [])
        self.RR2.assign_one_instrument_model_to_instrument_device_with_has_model(y, x)
        self.rr.create_association.assert_called_once_with(x, PRED.hasModel, y)

        rst()
        self.rr.find_objects.return_value = (["a", "b"], ["c", "d"])
        self.assertRaises(Inconsistent, self.RR2.assign_one_instrument_model_to_instrument_device_with_has_model, y, x)

        rst()
        self.rr.find_objects.return_value = (["a"], ["b"])
        self.rr.get_association.return_value = "yay"
        self.RR2.assign_one_instrument_model_to_instrument_device_with_has_model(y, x)

        rst()
        self.rr.find_objects.return_value = (["a"], ["b"])
        self.rr.get_association.side_effect = NotFound("")
        self.assertRaises(BadRequest, self.RR2.assign_one_instrument_model_to_instrument_device_with_has_model, y, x)

    def test_assign_single_subject(self):
        x = "x_id"
        y = "y_id"

        def rst():
            self.rr.find_subjects.reset_mock()
            self.rr.get_association.reset_mock()


        rst()
        self.rr.find_subjects.return_value = ([], [])
        self.RR2.assign_instrument_device_to_one_instrument_site_with_has_device(y, x)
        self.rr.create_association.assert_called_once_with(x, PRED.hasDevice, y)

        rst()
        self.rr.find_subjects.return_value = (["a", "b"], ["c", "d"])
        self.assertRaises(Inconsistent, self.RR2.assign_instrument_device_to_one_instrument_site_with_has_device, y, x)

        rst()
        self.rr.find_subjects.return_value = (["a"], ["b"])
        self.rr.get_association.return_value = "yay"
        self.RR2.assign_instrument_device_to_one_instrument_site_with_has_device(y, x)

        rst()
        self.rr.find_subjects.return_value = (["a"], ["b"])
        self.rr.get_association.side_effect = NotFound("")
        self.assertRaises(BadRequest, self.RR2.assign_instrument_device_to_one_instrument_site_with_has_device, y, x)



    def test_bad_dynamics(self):
        x = "x_id"
        self.RR2.assign_foo_to_bar(x)
        self.rr.assign_foo_to_bar.assert_called_once_with(x)

        self.assertRaises(BadRequest, getattr, self.RR2, "find_instrument_model_of_instrument_device_using_has_site")
        self.assertRaises(BadRequest, getattr, self.RR2, "find_instrument_model_of_instrument_device_using_has_banana")
        #self.assertRaises(BadRequest, getattr, self.RR2, "find_data_product_of_data_process")

        self.RR2.find_sensor_model_by_data_product(x)
        self.rr.find_sensor_model_by_data_product.assert_called_once_with(x)



    def test_cached_predicate_search(self):
        d = "d_id"
        m = "m_id"
        x = "x_id"

        good_assn = DotDict(s=d, st=RT.InstrumentDevice, p=PRED.hasModel, o=m, ot=RT.InstrumentModel)
        bad_assn  = DotDict(s=d, st=RT.PlatformDevice, p=PRED.hasModel, o=m, ot=RT.PlatformModel)

        self.rr.find_associations.return_value = [good_assn, bad_assn]

        self.RR2.cache_predicate(PRED.hasModel)

        self.assertTrue(self.RR2.has_cached_predicate(PRED.hasModel))
        self.rr.find_associations.assert_called_once_with(predicate=PRED.hasModel, id_only=False)

        # object searches that should return 0, 0, 1 results
        results = self.RR2.find_objects(x, PRED.hasModel, RT.InstrumentModel, True)
        self.assertEqual([], results)
        results = self.RR2.find_instrument_model_ids_of_instrument_device_using_has_model(x)
        self.assertEqual([], results)
        results = self.RR2.find_instrument_model_ids_of_instrument_device_using_has_model(d)
        self.assertEqual([m], results)

        self.assertEqual(0, self.rr.find_objects.call_count)

        # subject searches that should return 0, 0, 1 results
        results = self.RR2.find_subjects(RT.InstrumentDevice, PRED.hasModel, x, True)
        self.assertEqual([], results)
        results = self.RR2.find_instrument_device_ids_by_instrument_model_using_has_model(x)
        self.assertEqual([], results)
        results = self.RR2.find_instrument_device_ids_by_instrument_model_using_has_model(m)
        self.assertEqual([d], results)

        self.assertEqual(0, self.rr.find_subjects.call_count)

# 需要导入模块: from ion.util.enhanced_resource_registry_client import EnhancedResourceRegistryClient [as 别名]
# 或者: from ion.util.enhanced_resource_registry_client.EnhancedResourceRegistryClient import cache_predicate [as 别名]
class RollXBuilder(object):
    """
    for rollups and rolldowns
    """

    def __init__(self, process=None):
        """
        the process should be the "self" of a service instance
        """
        assert process
        self.process = process
        self.RR2 = EnhancedResourceRegistryClient(self.process.clients.resource_registry)

    def get_toplevel_platformsite(self, site_id):
        if not self.RR2.has_cached_predicate(PRED.hasSite):
            self.RR2.cache_predicate(PRED.hasSite)

        parent_ids = self.RR2.find_platform_site_ids_by_platform_site_using_has_site(site_id)

        if 0 == len(parent_ids):
            return site_id  # assume this to be the top level

        return self.get_toplevel_platformsite(parent_ids[0])

    def get_parent_network_nodes(self, site_id):
        """
        return the parent nodes of this network node, including the given node
        """
        if not self.RR2.has_cached_predicate(PRED.hasNetworkParent):
            self.RR2.cache_predicate(PRED.hasNetworkParent)

        def get_h(acc, some_id):
            acc.append(some_id)
            parent_ids = self.RR2.find_platform_device_ids_of_platform_device_using_has_network_parent(site_id)
            if 0 == len(parent_ids):
                return acc
            return get_h(acc, parent_ids[0])

        return get_h([], site_id)

    def get_toplevel_network_node(self, device_id):
        if not self.RR2.has_cached_predicate(PRED.hasNetworkParent):
            self.RR2.cache_predicate(PRED.hasNetworkParent)

        parent_ids = self.RR2.find_platform_device_ids_of_platform_device_using_has_network_parent(device_id)

        if 0 == len(parent_ids):
            # it can only be the network parent if it has this association
            if 0 < len(self.RR2.find_platform_device_ids_by_platform_device_using_has_network_parent(device_id)):
                return device_id  # assume this to be top level
            else:
                return None

        return self.get_toplevel_network_node(parent_ids[0])

    def get_site_hierarchy(self, site_id, site_val_fn):
        """
        return (child_sites, site_ancestors)
        where child_sites is a dict mapping all child site ids to the value of site_val_fn(child_site_id)
          and site_ancestors is a dict mapping all site ids to a list of their hasSite children.
        """

        if not self.RR2.has_cached_predicate(PRED.hasSite):
            self.RR2.cache_predicate(PRED.hasSite)

        full_list = [site_id]
        acc = {}

        def _get_ancestors_h(s_id):
            s_child_ids = self.RR2.find_objects(s_id, PRED.hasSite, id_only=True)
            if s_child_ids:
                acc[s_id] = s_child_ids
            for scid in s_child_ids:
                _get_ancestors_h(scid)
                full_list.append(scid)

        _get_ancestors_h(site_id)

        return dict([(s, site_val_fn(s)) for s in full_list]), acc

    def get_network_hierarchy(self, device_id, device_val_fn):
        """
        return (child_devices, device_ancestors)
        where child_devices is a dict mapping all child device ids to the value of device_val_fn(child_device_id)
          and device_ancestors is a dict mapping all device ids to a list of their children as per
        """

        if not self.RR2.has_cached_predicate(PRED.hasNetworkParent):
            self.RR2.cache_predicate(PRED.hasNetworkParent)

        full_list = [device_id]
        acc = {}

        def _get_ancestors_h(d_id):
            d_child_ids = self.RR2.find_subjects(d_id, PRED.hasNetworkParent, id_only=True)
            if d_child_ids:
                acc[d_id] = d_child_ids
            for dcid in d_child_ids:
                _get_ancestors_h(dcid)
                full_list.append(dcid)
#.........这里部分代码省略.........

# 需要导入模块: from ion.util.enhanced_resource_registry_client import EnhancedResourceRegistryClient [as 别名]
# 或者: from ion.util.enhanced_resource_registry_client.EnhancedResourceRegistryClient import cache_predicate [as 别名]
    def _get_site_extension(self, site_id='', ext_associations=None, ext_exclude=None, user_id=''):
        """Returns an InstrumentDeviceExtension object containing additional related information

        @param site_id    str
        @param ext_associations    dict
        @param ext_exclude    list
        @retval observatory    ObservatoryExtension
        @throws BadRequest    A parameter is missing
        @throws NotFound    An object with the specified observatory_id does not exist
        """

        if not site_id:
            raise BadRequest("The site_id parameter is empty")

        extended_resource_handler = ExtendedResourceContainer(self)

        extended_site = extended_resource_handler.create_extended_resource_container(
            extended_resource_type=OT.SiteExtension,
            resource_id=site_id,
            computed_resource_type=OT.SiteComputedAttributes,
            ext_associations=ext_associations,
            ext_exclude=ext_exclude,
            user_id=user_id)

        RR2 = EnhancedResourceRegistryClient(self.RR)
        RR2.cache_predicate(PRED.hasModel)

        # Get status of Site instruments.
        a, b =  self._get_instrument_states(extended_site.instrument_devices)
        extended_site.instruments_operational, extended_site.instruments_not_operational = a, b

        # lookup all hasModel predicates
        # lookup is a 2d associative array of [subject type][subject id] -> object id
        lookup = dict([(rt, {}) for rt in [RT.InstrumentDevice, RT.PlatformDevice]])
        for a in RR2.filter_cached_associations(PRED.hasModel, lambda assn: assn.st in lookup):
            lookup[a.st][a.s] = a.o

        def retrieve_model_objs(rsrc_list, object_type):
        # rsrc_list is devices that need models looked up.  object_type is the resource type (a device)
        # not all devices have models (represented as None), which kills read_mult.  so, extract the models ids,
        #  look up all the model ids, then create the proper output
            model_list = [lookup[object_type].get(r._id) for r in rsrc_list]
            model_uniq = list(set([m for m in model_list if m is not None]))
            model_objs = self.RR2.read_mult(model_uniq)
            model_dict = dict(zip(model_uniq, model_objs))
            return [model_dict.get(m) for m in model_list]

        extended_site.instrument_models = retrieve_model_objs(extended_site.instrument_devices, RT.InstrumentDevice)
        extended_site.platform_models   = retrieve_model_objs(extended_site.platform_devices, RT.PlatformDevice)


        # Status computation
        extended_site.computed.instrument_status = [AgentStatusBuilder.get_aggregate_status_of_device(idev._id, "aggstatus")
                                                    for idev in extended_site.instrument_devices]
        extended_site.computed.platform_status   = [AgentStatusBuilder.get_aggregate_status_of_device(pdev._id, "aggstatus")
                                                    for pdev in extended_site.platform_devices]

#            AgentStatusBuilder.add_device_aggregate_status_to_resource_extension(device_id,
#                                                                                    'aggstatus',
#                                                                                    extended_site)
        def status_unknown():
            return ComputedIntValue(status=ComputedValueAvailability.PROVIDED, value=StatusType.STATUS_UNKNOWN)
        extended_site.computed.communications_status_roll_up = status_unknown()
        extended_site.computed.power_status_roll_up          = status_unknown()
        extended_site.computed.data_status_roll_up           = status_unknown()
        extended_site.computed.location_status_roll_up       = status_unknown()
        extended_site.computed.aggregated_status             = status_unknown()

        extended_site.computed.site_status = [StatusType.STATUS_UNKNOWN] * len(extended_site.sites)



        return extended_site, RR2