Python loader.ContentPackLoader类代码示例

def _get_user_sensors():
    sensors_dict = defaultdict(list)
    pack_loader = ContentPackLoader()
    sensor_loader = SensorLoader()
    packs = pack_loader.get_content(base_dir=cfg.CONF.content.packs_base_path,
                                    content_type='sensors')
    for pack, sensor_dir in six.iteritems(packs):
        try:
            LOG.info('Loading sensors from pack: %s, dir: %s', pack, sensor_dir)
            if _is_requirements_ok(sensor_dir):
                base_dir = os.path.realpath(sensor_dir)
                pack_sensors = sensor_loader.get_sensors(base_dir=base_dir)

                # Include content pack name on the sensor class
                # TODO: This is nasty
                pack_sensors_augmented = defaultdict(list)
                for filename, sensors in pack_sensors.iteritems():
                    for sensor in sensors:
                        sensor.pack = pack
                        pack_sensors_augmented[filename].append(sensor)

                sensors_dict.update(pack_sensors_augmented)
            else:
                LOG.warning('Not registering sensors in sensor_dir: %s.', sensor_dir)
        except:
            LOG.exception('Failed loading sensors from dir: %s' % sensor_dir)
    return sensors_dict

def _get_api_models_from_disk(artifact_type, pack_dir=None):
    loader = ContentPackLoader()
    artifacts = None

    if pack_dir:
        artifacts_dir = loader.get_content_from_pack(pack_dir, artifact_type)
        pack_name = os.path.basename(os.path.normpath(pack_dir))
        artifacts = {pack_name: artifacts_dir}
    else:
        packs_dirs = content_utils.get_packs_base_paths()
        artifacts = loader.get_content(packs_dirs, artifact_type)

    artifacts_dict = {}
    for pack_name, pack_path in artifacts.items():
        artifacts_paths = registrar.get_resources_from_pack(pack_path)
        for artifact_path in artifacts_paths:
            artifact = meta_loader.load(artifact_path)
            if artifact_type == "sensors":
                sensors_dir = os.path.dirname(artifact_path)
                sensor_file_path = os.path.join(sensors_dir, artifact["entry_point"])
                artifact["artifact_uri"] = "file://" + sensor_file_path
            name = artifact.get("name", None) or artifact.get("class_name", None)
            if not artifact.get("pack", None):
                artifact["pack"] = pack_name
            ref = ResourceReference.to_string_reference(name=name, pack=pack_name)
            API_MODEL = API_MODELS_ARTIFACT_TYPES[artifact_type]
            # Following conversions are required because we add some fields with
            # default values in db model. If we don't do these conversions,
            # we'll see a unnecessary diff for those fields.
            artifact_api = API_MODEL(**artifact)
            artifact_db = API_MODEL.to_model(artifact_api)
            artifact_api = API_MODEL.from_model(artifact_db)
            artifacts_dict[ref] = artifact_api

    return artifacts_dict

    def _get_action_alias_db_by_name(self, name):
        """
        Retrieve ActionAlias DB object for the provided alias name.
        """
        base_pack_path = self._get_base_pack_path()
        pack_yaml_path = os.path.join(base_pack_path, MANIFEST_FILE_NAME)

        if os.path.isfile(pack_yaml_path):
            # 1. 1st try to infer pack name from pack metadata file
            meta_loader = MetaLoader()
            pack_metadata = meta_loader.load(pack_yaml_path)
            pack = get_pack_ref_from_metadata(metadata=pack_metadata)
        else:
            # 2. If pack.yaml is not available, fail back to directory name
            # Note: For this to work, directory name needs to match pack name
            _, pack = os.path.split(base_pack_path)

        pack_loader = ContentPackLoader()
        registrar = AliasesRegistrar(use_pack_cache=False)

        aliases_path = pack_loader.get_content_from_pack(pack_dir=base_pack_path,
                                                         content_type='aliases')
        aliases = registrar._get_aliases_from_pack(aliases_dir=aliases_path)
        for alias_path in aliases:
            action_alias_db = registrar._get_action_alias_db(pack=pack,
                                                             action_alias=alias_path,
                                                             ignore_metadata_file_error=True)

            if action_alias_db.name == name:
                return action_alias_db

        raise ValueError('Alias with name "%s" not found' % (name))

 def test_invalid_content_type(self):
     packs_base_path = os.path.join(
         os.path.dirname(os.path.realpath(__file__)), 'resources/packs/')
     loader = ContentPackLoader()
     try:
         loader.get_content(base_dir=packs_base_path, content_type='stuff')
         self.fail('Asking for invalid content should have thrown.')
     except:
         pass

 def register_actions_from_packs(self, base_dir):
     pack_loader = ContentPackLoader()
     dirs = pack_loader.get_content(base_dir=base_dir,
                                    content_type='actions')
     for pack, actions_dir in six.iteritems(dirs):
         try:
             actions = self._get_actions_from_pack(actions_dir)
             self._register_actions_from_pack(pack, actions)
         except:
             LOG.exception('Failed registering all actions from pack: %s', actions_dir)

 def test_get_sensors_pack_missing_sensors(self):
     loader = ContentPackLoader()
     fail_pack_path = os.path.join(
         os.path.dirname(os.path.realpath(__file__)), 'resources/packs/pack2')
     self.assertTrue(os.path.exists(fail_pack_path))
     try:
         loader._get_sensors(fail_pack_path)
         self.fail('Empty packs must throw exception.')
     except:
         pass

 def register_rules_from_packs(self, base_dir):
     pack_loader = ContentPackLoader()
     dirs = pack_loader.get_content(base_dir=base_dir,
                                    content_type='rules')
     for pack, rules_dir in six.iteritems(dirs):
         try:
             LOG.info('Registering rules from pack: %s', pack)
             rules = self._get_rules_from_pack(rules_dir)
             self._register_rules_from_pack(pack, rules)
         except:
             LOG.exception('Failed registering all rules from pack: %s', rules_dir)

    def register_sensors_from_packs(self, base_dir):
        pack_loader = ContentPackLoader()
        dirs = pack_loader.get_content(base_dir=base_dir, content_type='sensors')

        # Add system sensors to the core pack
        dirs['core'] = {}
        dirs['core'] = SYSTEM_SENSORS_PATH

        for pack, sensors_dir in six.iteritems(dirs):
            try:
                LOG.info('Registering sensors from pack: %s', pack)
                sensors = self._get_sensors_from_pack(sensors_dir)
                self._register_sensors_from_pack(pack=pack, sensors=sensors)
            except Exception as e:
                LOG.exception('Failed registering all sensors from pack "%s": %s', sensors_dir,
                              str(e))

    def test_get_content_multiple_directories(self):
        packs_base_path_1 = os.path.join(RESOURCES_DIR, 'packs/')
        packs_base_path_2 = os.path.join(RESOURCES_DIR, 'packs2/')
        base_dirs = [packs_base_path_1, packs_base_path_2]

        LOG.warning = Mock()

        loader = ContentPackLoader()
        sensors = loader.get_content(base_dirs=base_dirs, content_type='sensors')
        self.assertTrue('pack1' in sensors)  # from packs/
        self.assertTrue('pack3' in sensors)  # from packs2/

        # Assert that a warning is emitted when a duplicated pack is found
        expected_msg = ('Pack "pack1" already found in '
                        '"%s/packs/", ignoring content from '
                        '"%s/packs2/"' % (RESOURCES_DIR, RESOURCES_DIR))
        LOG.warning.assert_called_once_with(expected_msg)

 def __init__(self, use_pack_cache=True):
     """
     :param use_pack_cache: True to cache which packs have been registered in memory and making
                             sure packs are only registered once.
     :type use_pack_cache: ``bool``
     """
     self._use_pack_cache = use_pack_cache
     self._meta_loader = MetaLoader()
     self._pack_loader = ContentPackLoader()

    def _get_action_alias_db_by_name(self, name):
        """
        Retrieve ActionAlias DB object for the provided alias name.
        """
        base_pack_path = self._get_base_pack_path()
        _, pack = os.path.split(base_pack_path)

        pack_loader = ContentPackLoader()
        registrar = AliasesRegistrar(use_pack_cache=False)

        aliases_path = pack_loader.get_content_from_pack(pack_dir=base_pack_path,
                                                         content_type='aliases')
        aliases = registrar._get_aliases_from_pack(aliases_dir=aliases_path)
        for alias_path in aliases:
            action_alias_db = registrar._get_action_alias_db(pack=pack,
                                                             action_alias=alias_path)

            if action_alias_db.name == name:
                return action_alias_db

        raise ValueError('Alias with name "%s" not found' % (name))

    def __init__(self, use_pack_cache=True, fail_on_failure=False):
        """
        :param use_pack_cache: True to cache which packs have been registered in memory and making
                                sure packs are only registered once.
        :type use_pack_cache: ``bool``

        :param fail_on_failure: Throw an exception if resource registration fails.
        :type fail_on_failure: ``bool``
        """
        self._use_pack_cache = use_pack_cache
        self._fail_on_failure = fail_on_failure

        self._meta_loader = MetaLoader()
        self._pack_loader = ContentPackLoader()

    def __init__(self, use_pack_cache=True, use_runners_cache=False, fail_on_failure=False):
        """
        :param use_pack_cache: True to cache which packs have been registered in memory and making
                                sure packs are only registered once.
        :type use_pack_cache: ``bool``

        :param use_runners_cache: True to cache RunnerTypeDB objects in memory to reduce load on
                                  the database.
        :type use_runners_cache: ``bool``

        :param fail_on_failure: Throw an exception if resource registration fails.
        :type fail_on_failure: ``bool``
        """
        self._use_pack_cache = use_pack_cache
        self._use_runners_cache = use_runners_cache
        self._fail_on_failure = fail_on_failure

        self._meta_loader = MetaLoader()
        self._pack_loader = ContentPackLoader()

        # Maps runner name -> RunnerTypeDB
        self._runner_type_db_cache = {}

 def test_get_sensors_pack_missing_sensors(self):
     loader = ContentPackLoader()
     fail_pack_path = os.path.join(RESOURCES_DIR, 'packs/pack2')
     self.assertTrue(os.path.exists(fail_pack_path))
     self.assertEqual(loader._get_sensors(fail_pack_path), None)

class ResourceRegistrar(object):
    ALLOWED_EXTENSIONS = []

    def __init__(self, use_pack_cache=True):
        """
        :param use_pack_cache: True to cache which packs have been registered in memory and making
                                sure packs are only registered once.
        :type use_pack_cache: ``bool``
        """
        self._use_pack_cache = use_pack_cache
        self._meta_loader = MetaLoader()
        self._pack_loader = ContentPackLoader()

    def get_resources_from_pack(self, resources_dir):
        resources = []
        for ext in self.ALLOWED_EXTENSIONS:
            resources_glob = resources_dir

            if resources_dir.endswith("/"):
                resources_glob = resources_dir + ext
            else:
                resources_glob = resources_dir + "/*" + ext

            resource_files = glob.glob(resources_glob)
            resources.extend(resource_files)

        resources = sorted(resources)
        return resources

    def register_packs(self, base_dirs):
        """
        Register packs in all the provided directories.
        """
        packs = self._pack_loader.get_packs(base_dirs=base_dirs)

        registered_count = 0
        for pack_name, pack_path in six.iteritems(packs):
            self.register_pack(pack_name=pack_name, pack_dir=pack_path)
            registered_count += 1

        return registered_count

    def register_pack(self, pack_name, pack_dir):
        """
        Register pack in the provided directory.
        """
        if self._use_pack_cache and pack_name in REGISTERED_PACKS_CACHE:
            # This pack has already been registered during this register content run
            return

        LOG.debug("Registering pack: %s" % (pack_name))
        REGISTERED_PACKS_CACHE[pack_name] = True

        try:
            pack_db = self._register_pack(pack_name=pack_name, pack_dir=pack_dir)
        except Exception:
            LOG.exception('Failed to register pack "%s"' % (pack_name))
            return None

        return pack_db

    def _register_pack(self, pack_name, pack_dir):
        """
        Register a pack (create a DB object in the system).

        Note: Pack registration now happens when registering the content and not when installing
        a pack using packs.install. Eventually this will be moved to the pack management API.
        """
        manifest_path = os.path.join(pack_dir, MANIFEST_FILE_NAME)

        if not os.path.isfile(manifest_path):
            raise ValueError('Pack "%s" is missing %s file' % (pack_name, MANIFEST_FILE_NAME))

        content = self._meta_loader.load(manifest_path)
        if not content:
            raise ValueError('Pack "%s" metadata file is empty' % (pack_name))

        content["ref"] = pack_name

        # Include a list of pack files
        pack_file_list = get_file_list(directory=pack_dir, exclude_patterns=EXCLUDE_FILE_PATTERNS)
        content["files"] = pack_file_list

        pack_api = PackAPI(**content)
        pack_db = PackAPI.to_model(pack_api)

        try:
            pack_db.id = Pack.get_by_ref(pack_name).id
        except ValueError:
            LOG.debug("Pack %s not found. Creating new one.", pack_name)

        pack_db = Pack.add_or_update(pack_db)
        LOG.debug("Pack %s registered." % (pack_name))
        return pack_db