Python ResourceRegistryServiceProcessClient.find_resources方法代码示例

# 需要导入模块: from interface.services.coi.iresource_registry_service import ResourceRegistryServiceProcessClient [as 别名]
# 或者: from interface.services.coi.iresource_registry_service.ResourceRegistryServiceProcessClient import find_resources [as 别名]
    def delete_indexes(self):

        ims_cli = IndexManagementServiceProcessClient(process=self)
        rr_cli  = ResourceRegistryServiceProcessClient(process=self)

        #--------------------------------------------------------------------------------
        # Standard Indexes
        #--------------------------------------------------------------------------------

        for index,resources in STD_INDEXES.iteritems():
            index_ids, _ = rr_cli.find_resources(name=index,restype=RT.ElasticSearchIndex, id_only=True)
            for index_id in index_ids:
                ims_cli.delete_index(index_id)

        #--------------------------------------------------------------------------------
        # CouchDB Indexes
        #--------------------------------------------------------------------------------
        
        for index,datastore in COUCHDB_INDEXES.iteritems():
            index_ids, _ = rr_cli.find_resources(name=index, restype=RT.CouchDBIndex, id_only=True)
            for index_id in index_ids:
                ims_cli.delete_index(index_id)

        #--------------------------------------------------------------------------------
        # Edge Indexes
        #--------------------------------------------------------------------------------

        index_ids, _ = rr_cli.find_resources(name='%s_resources_index' % self.sysname,restype=RT.ElasticSearchIndex, id_only=True)
        for index_id in index_ids:
            ims_cli.delete_index(index_id)

        index_ids, _ = rr_cli.find_resources(name='%s_events_index' % self.sysname, restype=RT.ElasticSearchIndex, id_only=True)
        for index_id in index_ids:
            ims_cli.delete_index(index_id)

# 需要导入模块: from interface.services.coi.iresource_registry_service import ResourceRegistryServiceProcessClient [as 别名]
# 或者: from interface.services.coi.iresource_registry_service.ResourceRegistryServiceProcessClient import find_resources [as 别名]
def list_resources_by_type(resource_type):

    result = None

    client = ResourceRegistryServiceProcessClient(node=Container.instance.node, process=service_gateway_instance)
    try:
        #Resource Types are not in unicode
        res_list,_ = client.find_resources(restype=convert_unicode(resource_type) )

        return json_response({ GATEWAY_RESPONSE :res_list } )

    except Exception, e:
        return build_error_response(e)

# 需要导入模块: from interface.services.coi.iresource_registry_service import ResourceRegistryServiceProcessClient [as 别名]
# 或者: from interface.services.coi.iresource_registry_service.ResourceRegistryServiceProcessClient import find_resources [as 别名]
def list_resources_by_type(resource_type):

    ret = None

    client = ResourceRegistryServiceProcessClient(node=Container.instance.node, process=service_gateway_instance)
    try:
        res_list,_ = client.find_resources(restype=resource_type )
        result = []
        for res in res_list:
            result.append(res)

        ret = simplejson.dumps(result, default=ion_object_encoder)
    except Exception, e:
        ret =  "Error: %s" % e

# 需要导入模块: from interface.services.coi.iresource_registry_service import ResourceRegistryServiceProcessClient [as 别名]
# 或者: from interface.services.coi.iresource_registry_service.ResourceRegistryServiceProcessClient import find_resources [as 别名]
def find_resources_by_type(resource_type):
    try:
        client = ResourceRegistryServiceProcessClient(node=Container.instance.node, process=service_gateway_instance)

        #Validate requesting user and expiry and add governance headers
        ion_actor_id, expiry = get_governance_info_from_request()
        ion_actor_id, expiry = validate_request(ion_actor_id, expiry)

        #Resource Types are not in unicode
        res_list,_ = client.find_resources(restype=convert_unicode(resource_type) )

        return gateway_json_response(res_list)

    except Exception, e:
        return build_error_response(e)

# 需要导入模块: from interface.services.coi.iresource_registry_service import ResourceRegistryServiceProcessClient [as 别名]
# 或者: from interface.services.coi.iresource_registry_service.ResourceRegistryServiceProcessClient import find_resources [as 别名]
class ExchangeManager(object):
    """
    Manager object for the CC to manage Exchange related resources.
    """
    def __init__(self, container):
        log.debug("ExchangeManager initializing ...")
        self.container = container

        # Define the callables that can be added to Container public API
        # @TODO: remove
        self.container_api = [self.create_xs,
                              self.create_xp,
                              self.create_xn_service,
                              self.create_xn_process,
                              self.create_xn_queue]

        # Add the public callables to Container
        for call in self.container_api:
            setattr(self.container, call.__name__, call)

        self.default_xs         = ExchangeSpace(self, ION_ROOT_XS)
        self._xs_cache          = {}        # caching of xs names to RR objects
        self._default_xs_obj    = None      # default XS registry object
        self.org_id             = None

        # mappings
        self.xs_by_name = { ION_ROOT_XS: self.default_xs }      # friendly named XS to XSO
        self.xn_by_name = {}                                    # friendly named XN to XNO
        # xn by xs is a property

        self._chan = None

        # @TODO specify our own to_name here so we don't get auto-behavior - tricky chicken/egg
        self._ems_client    = ExchangeManagementServiceProcessClient(process=self.container)
        self._rr_client     = ResourceRegistryServiceProcessClient(process=self.container)

        # TODO: Do more initializing here, not in container

    @property
    def xn_by_xs(self):
        """
        Get a list of XNs associated by XS (friendly name).
        """
        ret = {}
        for xnname, xn in self.xn_by_name.iteritems():
            xsn = xn._xs._exchange
            if not xsn in ret:
                ret[xsn] = []
            ret[xsn].append(xn)

        return ret

    def _get_xs_obj(self, name=ION_ROOT_XS):
        """
        Gets a resource-registry represented XS, either via cache or RR request.
        """
        if name in self._xs_cache:
            return self._xs_cache[name]

        xs_objs, _ = self._rr_client.find_resources(RT.ExchangeSpace, name=name)
        if not len(xs_objs) == 1:
            log.warn("Could not find RR XS object with name: %s", name)
            return None

        self._xs_cache[name] = xs_objs[0]
        return xs_objs[0]

    def start(self):
        log.debug("ExchangeManager starting ...")

        # Establish connection to broker
        # @TODO: raise error if sux
        node, ioloop = messaging.make_node()

        self._transport = AMQPTransport.get_instance()
        self._client    = self._get_channel(node)

        # Declare root exchange
        #self.default_xs.ensure_exists(self._get_channel())
        return node, ioloop

    def _ems_available(self):
        """
        Returns True if the EMS is (likely) available and the auto_register CFG entry is True.

        Has the side effect of bootstrapping the org_id and default_xs's id/rev from the RR.
        Therefore, cannot be a property.
        """
        if CFG.container.get('exchange', {}).get('auto_register', False):
            # ok now make sure it's in the directory
            svc_de = self.container.directory.lookup('/Services/exchange_management')
            if svc_de is not None:
                if not self.org_id:
                    # find the default Org
                    org_ids = self._rr_client.find_resources(RT.Org, id_only=True)
                    if not (len(org_ids) and len(org_ids[0]) == 1):
                        log.warn("EMS available but could not find Org")
                        return False

                    self.org_id = org_ids[0][0]
#.........这里部分代码省略.........

# 需要导入模块: from interface.services.coi.iresource_registry_service import ResourceRegistryServiceProcessClient [as 别名]
# 或者: from interface.services.coi.iresource_registry_service.ResourceRegistryServiceProcessClient import find_resources [as 别名]

#.........这里部分代码省略.........
        assertions(len(workflow_output_ids) == 1 )

        log.debug( "persisting the output product")
        #self.dataproductclient.activate_data_product_persistence(workflow_product_id)
        dataset_ids,_ = self.rrclient.find_objects(workflow_product_id, PRED.hasDataset, RT.Dataset, True)
        assertions(len(dataset_ids) == 1 )
        dataset_id = dataset_ids[0]

        log.debug( "Verifying the output data product name matches what was specified in the workflow definition")
        workflow_product = self.rrclient.read(workflow_product_id)
        assertions(workflow_product.name.startswith(workflow_data_product_name), 'Nope: %s != %s' % (workflow_product.name, workflow_data_product_name))

        log.debug( "Walking the associations to find the appropriate output data streams to validate the messages")

        workflow_dp_ids,_ = self.rrclient.find_objects(workflow_id, PRED.hasDataProduct, RT.DataProduct, True)
        assertions(len(workflow_dp_ids) == 2 )

        for dp_id in workflow_dp_ids:
            stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True)
            assertions(len(stream_ids) == 1 )
            data_product_stream_ids.append(stream_ids[0])

        log.debug( "data_product_stream_ids: %s" % data_product_stream_ids)

        log.debug( "Starting the output stream listener to monitor to collect messages")
        results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids)

        log.debug( "results::: %s" % results)

        log.debug( "Stopping the workflow processes")
        self.workflowclient.terminate_data_process_workflow(workflow_id, False, timeout=250)  # Should test true at some point

        log.debug( "Making sure the Workflow object was removed")
        objs, _ = self.rrclient.find_resources(restype=RT.Workflow)
        assertions(len(objs) == 0)

        log.debug( "Validating the data from each of the messages along the way")
        self.validate_messages(results)

        log.debug( "Checking to see if dataset id = %s, was persisted, and that it can be retrieved...." % dataset_id)
        self.validate_data_ingest_retrieve(dataset_id)

        log.debug( "Cleaning up to make sure delete is correct.")
        self.workflowclient.delete_workflow_definition(workflow_def_id)

        workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition)
        assertions(len(workflow_def_ids) == 0 )



    @attr('LOCOINT')
    @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
    def test_google_dt_transform_workflow(self):

        assertions = self.assertTrue

        # Build the workflow definition
        workflow_def_obj = IonObject(RT.WorkflowDefinition, name='GoogleDT_Test_Workflow',description='Tests the workflow of converting stream data to Google DT')

        #Add a transformation process definition
        google_dt_procdef_id = self.create_google_dt_data_process_definition()
        workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=google_dt_procdef_id, persist_process_output_data=False)
        workflow_def_obj.workflow_steps.append(workflow_step_obj)

        #Create it in the resource registry
        workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj)

# 需要导入模块: from interface.services.coi.iresource_registry_service import ResourceRegistryServiceProcessClient [as 别名]
# 或者: from interface.services.coi.iresource_registry_service.ResourceRegistryServiceProcessClient import find_resources [as 别名]

#.........这里部分代码省略.........
    def default_node(self):
        """
        Returns the default node connection.
        """
        if 'primary' in self._nodes:
            return self._nodes['primary']
        elif len(self._nodes):
            log.warn("No primary connection, returning first available")
            return self._nodes.values()[0]

        return None

    @property
    def xn_by_xs(self):
        """
        Get a list of XNs associated by XS (friendly name).
        """
        ret = {}
        for xnname, xn in self.xn_by_name.iteritems():
            xsn = xn._xs._exchange
            if not xsn in ret:
                ret[xsn] = []
            ret[xsn].append(xn)

        return ret

    def _get_xs_obj(self, name=ION_ROOT_XS):
        """
        Gets a resource-registry represented XS, either via cache or RR request.
        """
        if name in self._xs_cache:
            return self._xs_cache[name]

        xs_objs, _ = self._rr_client.find_resources(RT.ExchangeSpace, name=name)
        if not len(xs_objs) == 1:
            log.warn("Could not find RR XS object with name: %s", name)
            return None

        self._xs_cache[name] = xs_objs[0]
        return xs_objs[0]

    def _ems_available(self):
        """
        Returns True if the EMS is (likely) available and the auto_register CFG entry is True.

        Has the side effect of bootstrapping the org_id and default_xs's id/rev from the RR.
        Therefore, cannot be a property.
        """
        if CFG.container.get('exchange', {}).get('auto_register', False):
            # ok now make sure it's in the directory
            svc_de = self.container.directory.lookup('/Services/exchange_management')
            if svc_de is not None:
                if not self.org_id:
                    # find the default Org
                    org_ids = self._rr_client.find_resources(RT.Org, id_only=True)
                    if not (len(org_ids) and len(org_ids[0]) == 1):
                        log.warn("EMS available but could not find Org")
                        return False

                    self.org_id = org_ids[0][0]
                    log.debug("Bootstrapped Container exchange manager with org id: %s", self.org_id)
                return True

        return False

    def _get_channel(self, node):

# 需要导入模块: from interface.services.coi.iresource_registry_service import ResourceRegistryServiceProcessClient [as 别名]
# 或者: from interface.services.coi.iresource_registry_service.ResourceRegistryServiceProcessClient import find_resources [as 别名]
class BootstrapProcessDispatcher(BootstrapPlugin):
    """
    Bootstrap process for process dispatcher.
    """

    def on_initial_bootstrap(self, process, config, **kwargs):
        self.pds_client = ProcessDispatcherServiceProcessClient(process=process)
        self.resource_registry = ResourceRegistryServiceProcessClient(process=process)
        self.ingestion_worker(process,config)
        self.replay_defs(process,config)
        self.notification_worker(process,config)
        self.registration_worker(process,config)
        self.pydap_server(process,config)

    def pydap_server(self, process, config):
        pydap_module = config.get_safe('bootstrap.processes.pydap.module', 'ion.processes.data.externalization.lightweight_pydap')
        pydap_class  = config.get_safe('bootstrap.processes.pydap.class', 'LightweightPyDAP')

        use_pydap = config.get_safe('bootstrap.launch_pydap', False)


        process_definition = ProcessDefinition(
                name = 'pydap_server',
                description = 'Lightweight WSGI Server for PyDAP')
        process_definition.executable['module'] = pydap_module
        process_definition.executable['class'] = pydap_class

        self._create_and_launch(process_definition,use_pydap)



    def registration_worker(self, process, config):
        res, meta = self.resource_registry.find_resources(name='registration_worker', restype=RT.ProcessDefinition)
        if len(res):
            return

        registration_module = config.get_safe('bootstrap.processes.registration.module', 'ion.processes.data.registration.registration_process')
        registration_class  = config.get_safe('bootstrap.processes.registration.class', 'RegistrationProcess')
        use_pydap = config.get_safe('bootstrap.use_pydap', False)


        process_definition = ProcessDefinition(
                name='registration_worker',
                description='For registering datasets with ERDDAP')
        process_definition.executable['module'] = registration_module
        process_definition.executable['class']  = registration_class


        self._create_and_launch(process_definition, use_pydap)

    def _create_and_launch(self, process_definition, conditional=True):
        proc_def_id = self.pds_client.create_process_definition(process_definition=process_definition)

        if conditional:

            process_res_id = self.pds_client.create_process(process_definition_id=proc_def_id)
            self.pds_client.schedule_process(process_definition_id=proc_def_id, process_id=process_res_id)

    def ingestion_worker(self, process, config):
        # ingestion
        ingestion_module    = config.get_safe('bootstrap.processes.ingestion.module','ion.processes.data.ingestion.science_granule_ingestion_worker')
        ingestion_class     = config.get_safe('bootstrap.processes.ingestion.class' ,'ScienceGranuleIngestionWorker')
        ingestion_datastore = config.get_safe('bootstrap.processes.ingestion.datastore_name', 'datasets')
        ingestion_queue     = config.get_safe('bootstrap.processes.ingestion.queue' , 'science_granule_ingestion')
        ingestion_workers   = config.get_safe('bootstrap.processes.ingestion.workers', 1)
        #--------------------------------------------------------------------------------
        # Create ingestion workers
        #--------------------------------------------------------------------------------

        process_definition = ProcessDefinition(
            name='ingestion_worker_process',
            description='Worker transform process for ingestion of datasets')
        process_definition.executable['module']= ingestion_module
        process_definition.executable['class'] = ingestion_class
        ingestion_procdef_id = self.pds_client.create_process_definition(process_definition=process_definition)

        #--------------------------------------------------------------------------------
        # Simulate a HA ingestion worker by creating two of them
        #--------------------------------------------------------------------------------
#        config = DotDict()
#        config.process.datastore_name = ingestion_datastore
#        config.process.queue_name     = ingestion_queue
#
#        for i in xrange(ingestion_workers):
#            self.pds_client.schedule_process(process_definition_id=ingestion_procdef_id, configuration=config)


    def notification_worker(self, process, config):
        # user notifications
        notification_module    = config.get_safe('bootstrap.processes.user_notification.module','ion.processes.data.transforms.notification_worker')
        notification_class     = config.get_safe('bootstrap.processes.user_notification.class' ,'NotificationWorker')
        notification_workers = config.get_safe('bootstrap.processes.user_notification.workers', 1)

        #--------------------------------------------------------------------------------
        # Create notification workers
        #--------------------------------------------------------------------------------

        # set up the process definition
        process_definition_uns = ProcessDefinition(
            name='notification_worker_process',
#.........这里部分代码省略.........

# 需要导入模块: from interface.services.coi.iresource_registry_service import ResourceRegistryServiceProcessClient [as 别名]
# 或者: from interface.services.coi.iresource_registry_service.ResourceRegistryServiceProcessClient import find_resources [as 别名]
class TestGovernanceInt(IonIntegrationTestCase):



    def setUp(self):

        # Start container
        self._start_container()

        #Load a deploy file
        self.container.start_rel_from_url('res/deploy/r2deploy.yml')

        #Instantiate a process to represent the test
        process=GovernanceTestProcess()


        #Load system policies after container has started all of the services
        LoadSystemPolicy.op_load_system_policies(process)

        self.rr_client = ResourceRegistryServiceProcessClient(node=self.container.node, process=process)

        self.id_client = IdentityManagementServiceProcessClient(node=self.container.node, process=process)

        self.pol_client = PolicyManagementServiceProcessClient(node=self.container.node, process=process)

        self.org_client = OrgManagementServiceProcessClient(node=self.container.node, process=process)

        self.ims_client = InstrumentManagementServiceProcessClient(node=self.container.node, process=process)

        self.ems_client = ExchangeManagementServiceProcessClient(node=self.container.node, process=process)

        self.ion_org = self.org_client.find_org()


        self.system_actor = self.id_client.find_actor_identity_by_name(name=CFG.system.system_actor)
        log.debug('system actor:' + self.system_actor._id)


        sa_header_roles = get_role_message_headers(self.org_client.find_all_roles_by_user(self.system_actor._id))
        self.sa_user_header = {'ion-actor-id': self.system_actor._id, 'ion-actor-roles': sa_header_roles }

    @attr('LOCOINT')
    @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
    def test_basic_policy(self):

        #Make sure that the system policies have been loaded
        policy_list,_ = self.rr_client.find_resources(restype=RT.Policy)
        self.assertNotEqual(len(policy_list),0,"The system policies have not been loaded into the Resource Registry")


        #Attempt to access an operation in service which does not have specific policies set
        es_obj = IonObject(RT.ExchangeSpace, description= 'ION test XS', name='ioncore2' )
        with self.assertRaises(Unauthorized) as cm:
            self.ems_client.create_exchange_space(es_obj)
        self.assertIn( 'exchange_management(create_exchange_space) has been denied',cm.exception.message)

        #Add a new policy to allow the the above service call.
        policy_obj = IonObject(RT.Policy, name='Exchange_Management_Test_Policy', definition_type="Service", rule=TEST_POLICY_TEXT,
            description='Allow specific operations in the Exchange Management Service for anonymous user')

        test_policy_id = self.pol_client.create_policy(policy_obj, headers=self.sa_user_header)
        self.pol_client.add_service_policy('exchange_management', test_policy_id, headers=self.sa_user_header)
        log.info('Policy created: ' + policy_obj.name)

        gevent.sleep(2)  # Wait for events to be fired and policy updated

        #The previous attempt at this operations should now be allowed.
        es_obj = IonObject(RT.ExchangeSpace, description= 'ION test XS', name='ioncore2' )
        with self.assertRaises(BadRequest) as cm:
            self.ems_client.create_exchange_space(es_obj)
        self.assertIn( 'Arguments not set',cm.exception.message)

        #disable the test policy to try again
        self.pol_client.disable_policy(test_policy_id, headers=self.sa_user_header)

        gevent.sleep(2)  # Wait for events to be fired and policy updated

        #The same request that previously was allowed should not be denied
        es_obj = IonObject(RT.ExchangeSpace, description= 'ION test XS', name='ioncore2' )
        with self.assertRaises(Unauthorized) as cm:
            self.ems_client.create_exchange_space(es_obj)
        self.assertIn( 'exchange_management(create_exchange_space) has been denied',cm.exception.message)

        #now enable the test policy to try again
        self.pol_client.enable_policy(test_policy_id, headers=self.sa_user_header)

        gevent.sleep(2)  # Wait for events to be fired and policy updated

        #The previous attempt at this operations should now be allowed.
        es_obj = IonObject(RT.ExchangeSpace, description= 'ION test XS', name='ioncore2' )
        with self.assertRaises(BadRequest) as cm:
            self.ems_client.create_exchange_space(es_obj)
        self.assertIn( 'Arguments not set',cm.exception.message)

        self.pol_client.remove_service_policy('exchange_management', test_policy_id, headers=self.sa_user_header)
        self.pol_client.delete_policy(test_policy_id, headers=self.sa_user_header)

        gevent.sleep(2)  # Wait for events to be fired and policy updated

        #The same request that previously was allowed should not be denied
#.........这里部分代码省略.........

# 需要导入模块: from interface.services.coi.iresource_registry_service import ResourceRegistryServiceProcessClient [as 别名]
# 或者: from interface.services.coi.iresource_registry_service.ResourceRegistryServiceProcessClient import find_resources [as 别名]
    org_client.grant_role(ion_org._id, user._id, 'INSTRUMENT_OPERATOR', headers={'ion-actor-id': system_actor._id, 'ion-actor-roles': header_roles })

    header_roles = get_role_message_headers(org_client.find_all_roles_by_user(user._id))

    ia_obj = IonObject(RT.InstrumentAgent, name='Instrument Agent1', description='The first Instrument Agent')

    ims_client.create_instrument_agent(ia_obj,  headers={'ion-actor-id': user._id, 'ion-actor-roles': header_roles })

    ia_obj = IonObject(RT.InstrumentAgent, name='Instrument Agent2', description='The second Instrument Agent')

    ims_client.create_instrument_agent(ia_obj, headers={'ion-actor-id': user._id, 'ion-actor-roles': header_roles })

    log.info( 'Instrument Agents')
    rr_client = ResourceRegistryServiceProcessClient(node=container.node, process=process)

    ia_list,_ = rr_client.find_resources(restype=RT.InstrumentAgent)
    for ia_obj in ia_list:
        log.info( str(ia_obj))

def test_requests(container, process=FakeProcess()):

    org_client = OrgManagementServiceProcessClient(node=container.node, process=process)
    ion_org = org_client.find_org()

    id_client = IdentityManagementServiceProcessClient(node=container.node, process=process)

    rr_client = ResourceRegistryServiceProcessClient(node=container.node, process=process)

    system_actor = id_client.find_actor_identity_by_name(name=CFG.system.system_actor)
    log.info('system actor:' + system_actor._id)