Python EnhancedResourceRegistryClient.find_resource_by_name方法代码示例

# 需要导入模块: from ion.util.enhanced_resource_registry_client import EnhancedResourceRegistryClient [as 别名]
# 或者: from ion.util.enhanced_resource_registry_client.EnhancedResourceRegistryClient import find_resource_by_name [as 别名]
class DMTestCase(IonIntegrationTestCase):
    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')
        self.resource_registry = self.container.resource_registry
        self.RR2 = EnhancedResourceRegistryClient(self.resource_registry)
        self.data_acquisition_management = DataAcquisitionManagementServiceClient()
        self.pubsub_management =  PubsubManagementServiceClient()
        self.instrument_management = InstrumentManagementServiceClient()
        self.data_product_management = DataProductManagementServiceClient()
        self.dataset_management =  DatasetManagementServiceClient()
        self.process_dispatcher = ProcessDispatcherServiceClient()
        self.data_process_management = DataProcessManagementServiceClient()
        self.data_product_management = DataProductManagementServiceClient()
        self.data_retriever = DataRetrieverServiceClient()
        self.dataset_management = DatasetManagementServiceClient()
        self.user_notification = UserNotificationServiceClient()
        self.workflow_management = WorkflowManagementServiceClient()
        self.visualization = VisualizationServiceClient()

    def create_stream_definition(self, *args, **kwargs):
        stream_def_id = self.pubsub_management.create_stream_definition(*args, **kwargs)
        self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
        return stream_def_id

    def create_data_product(self,name, stream_def_id='', param_dict_name='', pdict_id=''):
        if not (stream_def_id or param_dict_name or pdict_id):
            raise AssertionError('Attempted to create a Data Product without a parameter dictionary')

        tdom, sdom = time_series_domain()

        dp = DataProduct(name=name,
                spatial_domain = sdom.dump(),
                temporal_domain = tdom.dump(),
                )

        stream_def_id = stream_def_id or self.create_stream_definition('%s stream def' % name, 
                parameter_dictionary_id=pdict_id or self.RR2.find_resource_by_name(RT.ParameterDictionary,
                    param_dict_name, id_only=True))

        data_product_id = self.data_product_management.create_data_product(dp, stream_definition_id=stream_def_id)
        self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
        return data_product_id

    def activate_data_product(self, data_product_id):
        self.data_product_management.activate_data_product_persistence(data_product_id)
        self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id)
    
    def data_product_by_id(self, alt_id):
        data_products, _ = self.container.resource_registry.find_resources_ext(alt_id=alt_id, alt_id_ns='PRE', id_only=True)
        if data_products:
            return data_products[0]
        return None
    def dataset_of_data_product(self, data_product_id):
        return self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0]

# 需要导入模块: from ion.util.enhanced_resource_registry_client import EnhancedResourceRegistryClient [as 别名]
# 或者: from ion.util.enhanced_resource_registry_client.EnhancedResourceRegistryClient import find_resource_by_name [as 别名]

#.........这里部分代码省略.........

    def _find_streamdef_for_dp_and_pdict(self, dp_id, pdict_id):
        # Given a pdict_id and a data_product_id find the stream def in the middle
        pdict_stream_defs = self.RR2.find_stream_definition_ids_by_parameter_dictionary_using_has_parameter_dictionary(pdict_id)
        stream_def_id = self.RR2.find_stream_definition_id_of_data_product_using_has_stream_definition(dp_id)
        result = stream_def_id if stream_def_id in pdict_stream_defs else None

        return result


    def _generate_stream_config(self):
        log.debug("_generate_stream_config for %s", self.agent_instance_obj.name)
        dsm = self.clients.dataset_management
        psm = self.clients.pubsub_management

        agent_obj  = self._get_agent()
        device_obj = self._get_device()

        streams_dict = {}
        for stream_cfg in agent_obj.stream_configurations:
            #create a stream def for each param dict to match against the existing data products
            streams_dict[stream_cfg.stream_name] = {'param_dict_name':stream_cfg.parameter_dictionary_name}

        #retrieve the output products
        # TODO: What about platforms? other things?
        device_id = device_obj._id
        data_product_objs = self.RR2.find_data_products_of_instrument_device_using_has_output_product(device_id)

        stream_config = {}
        for dp in data_product_objs:
            stream_def_id = self.RR2.find_stream_definition_id_of_data_product_using_has_stream_definition(dp._id)
            for stream_name, stream_info_dict in streams_dict.items():
                # read objects from cache to be compared
                pdict = self.RR2.find_resource_by_name(RT.ParameterDictionary, stream_info_dict.get('param_dict_name'))
                stream_def_id = self._find_streamdef_for_dp_and_pdict(dp._id, pdict._id)

                if stream_def_id:
                    #model_param_dict = self.RR2.find_resources_by_name(RT.ParameterDictionary,
                    #                                         stream_info_dict.get('param_dict_name'))[0]
                    #model_param_dict = self._get_param_dict_by_name(stream_info_dict.get('param_dict_name'))
                    #stream_route = self.RR2.read(product_stream_id).stream_route
                    product_stream_id = self.RR2.find_stream_id_of_data_product_using_has_stream(dp._id)
                    stream_def = psm.read_stream_definition(stream_def_id)
                    stream_route = psm.read_stream_route(stream_id=product_stream_id)

                    from pyon.core.object import IonObjectSerializer
                    stream_def_dict = IonObjectSerializer().serialize(stream_def)
                    stream_def_dict.pop('type_')

                    if stream_name in stream_config:
                        log.warn("Overwriting stream_config[%s]", stream_name)

                    stream_config[stream_name] = {  'routing_key'           : stream_route.routing_key,  # TODO: Serialize stream_route together
                                                    'stream_id'             : product_stream_id,
                                                    'stream_definition_ref' : stream_def_id,
                                                    'stream_def_dict'       : stream_def_dict,  # This is very large
                                                    'exchange_point'        : stream_route.exchange_point,
                                                    # This is redundant and very large - the param dict is in the stream_def_dict
                                                    #'parameter_dictionary'  : stream_def.parameter_dictionary,

                    }
        if len(stream_config) < len(streams_dict):
            log.warn("Found only %s matching streams by stream definition (%s) than %s defined in the agent (%s).",
                     len(stream_config), stream_config.keys(), len(streams_dict), streams_dict.keys())

        log.debug("Stream config generated")

# 需要导入模块: from ion.util.enhanced_resource_registry_client import EnhancedResourceRegistryClient [as 别名]
# 或者: from ion.util.enhanced_resource_registry_client.EnhancedResourceRegistryClient import find_resource_by_name [as 别名]

#.........这里部分代码省略.........
        # Given a pdict_id and a data_product_id find the stream def in the middle
        pdict_stream_defs = self.RR2.find_stream_definition_ids_by_parameter_dictionary_using_has_parameter_dictionary(pdict_id)
        stream_def_id = self.RR2.find_stream_definition_id_of_data_product_using_has_stream_definition(dp_id)
        result = stream_def_id if stream_def_id in pdict_stream_defs else None
        return result


    def _generate_stream_config(self):
        log.debug("_generate_stream_config for %s", self.agent_instance_obj.name)
        dsm = self.clients.dataset_management
        psm = self.clients.pubsub_management

        agent_obj  = self._get_agent()
        device_obj = self._get_device()

        streams_dict = {}
        for stream_cfg in agent_obj.stream_configurations:
            #create a stream def for each param dict to match against the existing data products
            streams_dict[stream_cfg.stream_name] = {'param_dict_name':stream_cfg.parameter_dictionary_name,
                                                    #'stream_def_id':stream_def_id,
                                                    'records_per_granule': stream_cfg.records_per_granule,
                                                    'granule_publish_rate':stream_cfg.granule_publish_rate,
                                                     }

        #retrieve the output products
        device_id = device_obj._id
        data_product_objs = self.RR2.find_data_products_of_instrument_device_using_has_output_product(device_id)

        stream_config = {}
        for d in data_product_objs:
            stream_def_id = self.RR2.find_stream_definition_id_of_data_product_using_has_stream_definition(d._id)
            for model_stream_name, stream_info_dict  in streams_dict.items():
                # read objects from cache to be compared
                pdict = self.RR2.find_resource_by_name(RT.ParameterDictionary, stream_info_dict.get('param_dict_name'))
                stream_def_id = self._meet_in_the_middle(d._id, pdict._id)

                if stream_def_id:
                    #model_param_dict = self.RR2.find_resources_by_name(RT.ParameterDictionary,
                    #                                         stream_info_dict.get('param_dict_name'))[0]
                    #model_param_dict = self._get_param_dict_by_name(stream_info_dict.get('param_dict_name'))
                    #stream_route = self.RR2.read(product_stream_id).stream_route
                    product_stream_id = self.RR2.find_stream_id_of_data_product_using_has_stream(d._id)
                    stream_def = psm.read_stream_definition(stream_def_id)
                    stream_route = psm.read_stream_route(stream_id=product_stream_id)
                    
                    from pyon.core.object import IonObjectSerializer
                    stream_def_dict = IonObjectSerializer().serialize(stream_def)
                    sdtype = stream_def_dict.pop('type_')

                    if model_stream_name in stream_config:
                        log.warn("Overwiting stream_config[%s]", model_stream_name)

                    stream_config[model_stream_name] = {'routing_key'           : stream_route.routing_key,
                                                        'stream_id'             : product_stream_id,
                                                        'stream_definition_ref' : stream_def_id,
                                                        'stream_def_dict'       : stream_def_dict,
                                                        'exchange_point'        : stream_route.exchange_point,
                                                        'parameter_dictionary'  : stream_def.parameter_dictionary,
                                                        'records_per_granule'   : stream_info_dict.get('records_per_granule'),
                                                        'granule_publish_rate'  : stream_info_dict.get('granule_publish_rate'),
                    }

        log.debug("Stream config generated")
        log.trace("generate_stream_config: %s", str(stream_config) )
        return stream_config

# 需要导入模块: from ion.util.enhanced_resource_registry_client import EnhancedResourceRegistryClient [as 别名]
# 或者: from ion.util.enhanced_resource_registry_client.EnhancedResourceRegistryClient import find_resource_by_name [as 别名]
class DMTestCase(IonIntegrationTestCase):
    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url("res/deploy/r2deploy.yml")
        self.resource_registry = self.container.resource_registry
        self.RR2 = EnhancedResourceRegistryClient(self.resource_registry)
        self.data_acquisition_management = DataAcquisitionManagementServiceClient()
        self.pubsub_management = PubsubManagementServiceClient()
        self.instrument_management = InstrumentManagementServiceClient()
        self.discovery = DiscoveryServiceClient()
        self.dataset_management = DatasetManagementServiceClient()
        self.process_dispatcher = ProcessDispatcherServiceClient()
        self.data_process_management = DataProcessManagementServiceClient()
        self.data_product_management = DataProductManagementServiceClient()
        self.data_retriever = DataRetrieverServiceClient()
        self.dataset_management = DatasetManagementServiceClient()
        self.user_notification = UserNotificationServiceClient()
        self.workflow_management = WorkflowManagementServiceClient()
        self.observatory_management = ObservatoryManagementServiceClient()
        self.visualization = VisualizationServiceClient()
        self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
        self.ctd_count = 0

    def create_stream_definition(self, *args, **kwargs):
        stream_def_id = self.pubsub_management.create_stream_definition(*args, **kwargs)
        self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
        return stream_def_id

    def create_data_product(self, name, stream_def_id="", param_dict_name="", pdict_id=""):
        if not (stream_def_id or param_dict_name or pdict_id):
            raise AssertionError("Attempted to create a Data Product without a parameter dictionary")

        tdom, sdom = time_series_domain()

        dp = DataProduct(name=name, spatial_domain=sdom.dump(), temporal_domain=tdom.dump())

        stream_def_id = stream_def_id or self.create_stream_definition(
            "%s stream def" % name,
            parameter_dictionary_id=pdict_id
            or self.RR2.find_resource_by_name(RT.ParameterDictionary, param_dict_name, id_only=True),
        )

        data_product_id = self.data_product_management.create_data_product(dp, stream_definition_id=stream_def_id)
        self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
        return data_product_id

    def activate_data_product(self, data_product_id):
        self.data_product_management.activate_data_product_persistence(data_product_id)
        self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id)

    def data_product_by_id(self, alt_id):
        data_products, _ = self.container.resource_registry.find_resources_ext(
            alt_id=alt_id, alt_id_ns="PRE", id_only=True
        )
        if data_products:
            return data_products[0]
        return None

    def dataset_of_data_product(self, data_product_id):
        return self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0]

    def make_ctd_data_product(self):
        pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict")
        stream_def_id = self.create_stream_definition("ctd %d" % self.ctd_count, parameter_dictionary_id=pdict_id)
        data_product_id = self.create_data_product("ctd %d" % self.ctd_count, stream_def_id=stream_def_id)
        self.activate_data_product(data_product_id)
        self.ctd_count += 1
        return data_product_id

    def preload_ui(self):
        config = DotDict()
        config.op = "loadui"
        config.loadui = True
        config.attachments = "res/preload/r2_ioc/attachments"
        config.ui_path = "http://userexperience.oceanobservatories.org/database-exports/Candidates"

        self.container.spawn_process("preloader", "ion.processes.bootstrap.ion_loader", "IONLoader", config)

    def strap_erddap(self, data_product_id=None):
        """
        Copies the datasets.xml to /tmp
        """
        datasets_xml_path = RegistrationProcess.get_datasets_xml_path(CFG)
        if os.path.lexists("/tmp/datasets.xml"):
            os.unlink("/tmp/datasets.xml")
        os.symlink(datasets_xml_path, "/tmp/datasets.xml")
        if data_product_id:
            with open("/tmp/erddap/flag/data%s" % data_product_id, "a"):
                pass

        gevent.sleep(5)
        from subprocess import call

        call(["open", "http://localhost:9000/erddap/tabledap/data%s.html" % data_product_id])

    def launch_ui_facepage(self, data_product_id):
        """
        Opens the UI face page on localhost for a particular data product
        """
        from subprocess import call
#.........这里部分代码省略.........

# 需要导入模块: from ion.util.enhanced_resource_registry_client import EnhancedResourceRegistryClient [as 别名]
# 或者: from ion.util.enhanced_resource_registry_client.EnhancedResourceRegistryClient import find_resource_by_name [as 别名]
class DMTestCase(IonIntegrationTestCase):
    def setUp(self):
        self._start_container()
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')
        self.resource_registry = self.container.resource_registry
        self.RR2 = EnhancedResourceRegistryClient(self.resource_registry)
        self.data_acquisition_management = DataAcquisitionManagementServiceClient()
        self.pubsub_management =  PubsubManagementServiceClient()
        self.instrument_management = InstrumentManagementServiceClient()
        self.discovery = DiscoveryServiceClient()
        self.dataset_management =  DatasetManagementServiceClient()
        self.process_dispatcher = ProcessDispatcherServiceClient()
        self.data_process_management = DataProcessManagementServiceClient()
        self.data_product_management = DataProductManagementServiceClient()
        self.data_retriever = DataRetrieverServiceClient()
        self.dataset_management = DatasetManagementServiceClient()
        self.user_notification = UserNotificationServiceClient()
        self.workflow_management = WorkflowManagementServiceClient()
        self.observatory_management = ObservatoryManagementServiceClient()
        self.visualization = VisualizationServiceClient()
        self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
        self.ctd_count = 0

    def create_stream_definition(self, *args, **kwargs):
        stream_def_id = self.pubsub_management.create_stream_definition(*args, **kwargs)
        self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
        return stream_def_id

    def create_data_product(self,name, stream_def_id='', param_dict_name='', pdict_id='', stream_configuration=None):
        if not (stream_def_id or param_dict_name or pdict_id):
            raise AssertionError('Attempted to create a Data Product without a parameter dictionary')


        dp = DataProduct(name=name)

        stream_def_id = stream_def_id or self.create_stream_definition('%s stream def' % name, 
                parameter_dictionary_id=pdict_id or self.RR2.find_resource_by_name(RT.ParameterDictionary,
                    param_dict_name, id_only=True))


        stream_config = stream_configuration or StreamConfiguration(stream_name='parsed_ctd', stream_type=StreamConfigurationType.PARSED)

        data_product_id = self.data_product_management.create_data_product(dp, stream_definition_id=stream_def_id, default_stream_configuration=stream_config)
        self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
        return data_product_id

    def activate_data_product(self, data_product_id):
        self.data_product_management.activate_data_product_persistence(data_product_id)
        self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id)
    
    def data_product_by_id(self, alt_id):
        data_products, _ = self.container.resource_registry.find_resources_ext(alt_id=alt_id, alt_id_ns='PRE', id_only=True)
        if data_products:
            return data_products[0]
        return None
    def dataset_of_data_product(self, data_product_id):
        return self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0]
    
    def make_ctd_data_product(self):
        pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
        stream_def_id = self.create_stream_definition('ctd %d' % self.ctd_count, parameter_dictionary_id=pdict_id)
        data_product_id = self.create_data_product('ctd %d' % self.ctd_count, stream_def_id=stream_def_id)
        self.activate_data_product(data_product_id)
        self.ctd_count += 1
        return data_product_id
    
    def preload_ui(self):
        config = DotDict()
        config.op='loadui'
        config.loadui=True
        config.attachments='res/preload/r2_ioc/attachments'
        config.ui_path = "http://userexperience.oceanobservatories.org/database-exports/Candidates"
        
        self.container.spawn_process('preloader', 'ion.processes.bootstrap.ion_loader', 'IONLoader', config)
    
    def strap_erddap(self, data_product_id=None, open_page=True):
        '''
        Copies the datasets.xml to /tmp
        '''
        datasets_xml_path = RegistrationProcess.get_datasets_xml_path(CFG)
        if os.path.lexists('/tmp/datasets.xml'):
            os.unlink('/tmp/datasets.xml')
        os.symlink(datasets_xml_path, '/tmp/datasets.xml')
        if data_product_id:
            with open('/tmp/erddap/flag/data%s' % data_product_id, 'a'):
                pass

        if open_page:
            gevent.sleep(5)
            from subprocess import call
            call(['open', 'http://localhost:9000/erddap/tabledap/data%s.html' % data_product_id])
    
    def launch_ui_facepage(self, data_product_id):
        '''
        Opens the UI face page on localhost for a particular data product
        '''
        from subprocess import call
        call(['open', 'http://localhost:3000/DataProduct/face/%s/' % data_product_id])