Python utils.get_packs_base_paths函数代码示例

    def test_get_pack_base_paths(self):
        cfg.CONF.content.system_packs_base_path = ""
        cfg.CONF.content.packs_base_paths = "/opt/path1"
        result = get_packs_base_paths()
        self.assertEqual(result, ["/opt/path1"])

        # Multiple paths, no trailing colon
        cfg.CONF.content.packs_base_paths = "/opt/path1:/opt/path2"
        result = get_packs_base_paths()
        self.assertEqual(result, ["/opt/path1", "/opt/path2"])

        # Multiple paths, trailing colon
        cfg.CONF.content.packs_base_paths = "/opt/path1:/opt/path2:"
        result = get_packs_base_paths()
        self.assertEqual(result, ["/opt/path1", "/opt/path2"])

        # Multiple same paths
        cfg.CONF.content.packs_base_paths = "/opt/path1:/opt/path2:/opt/path1:/opt/path2"
        result = get_packs_base_paths()
        self.assertEqual(result, ["/opt/path1", "/opt/path2"])

        # Assert system path is always first
        cfg.CONF.content.system_packs_base_path = "/opt/system"
        cfg.CONF.content.packs_base_paths = "/opt/path2:/opt/path1"
        result = get_packs_base_paths()
        self.assertEqual(result, ["/opt/system", "/opt/path2", "/opt/path1"])

    def test_get_pack_base_paths(self):
        cfg.CONF.content.system_packs_base_path = ''
        cfg.CONF.content.packs_base_paths = '/opt/path1'
        result = get_packs_base_paths()
        self.assertEqual(result, ['/opt/path1'])

        # Multiple paths, no trailing colon
        cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2'
        result = get_packs_base_paths()
        self.assertEqual(result, ['/opt/path1', '/opt/path2'])

        # Multiple paths, trailing colon
        cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2:'
        result = get_packs_base_paths()
        self.assertEqual(result, ['/opt/path1', '/opt/path2'])

        # Multiple same paths
        cfg.CONF.content.packs_base_paths = '/opt/path1:/opt/path2:/opt/path1:/opt/path2'
        result = get_packs_base_paths()
        self.assertEqual(result, ['/opt/path1', '/opt/path2'])

        # Assert system path is always first
        cfg.CONF.content.system_packs_base_path = '/opt/system'
        cfg.CONF.content.packs_base_paths = '/opt/path2:/opt/path1'
        result = get_packs_base_paths()
        self.assertEqual(result, ['/opt/system', '/opt/path2', '/opt/path1'])

    def test_register_pack_pack_stackstorm_version_and_future_parameters(self):
        # Verify DB is empty
        pack_dbs = Pack.get_all()
        self.assertEqual(len(pack_dbs), 0)

        registrar = ResourceRegistrar(use_pack_cache=False)
        registrar._pack_loader.get_packs = mock.Mock()
        registrar._pack_loader.get_packs.return_value = {'dummy_pack_9': PACK_PATH_9}
        packs_base_paths = content_utils.get_packs_base_paths()
        registrar.register_packs(base_dirs=packs_base_paths)

        # Dependencies, stackstorm_version and future values
        pack_db = Pack.get_by_name('dummy_pack_9_deps')
        self.assertEqual(pack_db.dependencies, ['core=0.2.0'])
        self.assertEqual(pack_db.stackstorm_version, '>=1.6.0, <2.2.0')
        self.assertEqual(pack_db.system, {'centos': {'foo': '>= 1.0'}})
        self.assertEqual(pack_db.python_versions, ['2', '3'])

        # Note: We only store parameters which are defined in the schema, all other custom user
        # defined attributes are ignored
        self.assertTrue(not hasattr(pack_db, 'future'))
        self.assertTrue(not hasattr(pack_db, 'this'))

        # Wrong characters in the required st2 version
        expected_msg = "'wrongstackstormversion' does not match"
        self.assertRaisesRegexp(ValidationError, expected_msg, registrar._register_pack_db,
                                pack_name=None, pack_dir=PACK_PATH_10)

    def _register_packs(self):
        """
        Register all the packs inside the fixtures directory.
        """

        registrar = ResourceRegistrar(use_pack_cache=False)
        registrar.register_packs(base_dirs=get_packs_base_paths())

    def test_register_configs_for_all_packs(self):
        # Verify DB is empty
        pack_dbs = Pack.get_all()
        config_dbs = Config.get_all()

        self.assertEqual(len(pack_dbs), 0)
        self.assertEqual(len(config_dbs), 0)

        registrar = ConfigsRegistrar(use_pack_cache=False)
        registrar._pack_loader.get_packs = mock.Mock()
        registrar._pack_loader.get_packs.return_value = {'dummy_pack_1': PACK_1_PATH}
        packs_base_paths = content_utils.get_packs_base_paths()
        registrar.register_from_packs(base_dirs=packs_base_paths)

        # Verify pack and schema have been registered
        pack_dbs = Pack.get_all()
        config_dbs = Config.get_all()

        self.assertEqual(len(pack_dbs), 1)
        self.assertEqual(len(config_dbs), 1)

        config_db = config_dbs[0]
        self.assertEqual(config_db.values['api_key'], '{{st2kv.user.api_key}}')
        self.assertEqual(config_db.values['api_secret'], SUPER_SECRET_PARAMETER)
        self.assertEqual(config_db.values['region'], 'us-west-1')

def _get_api_models_from_disk(artifact_type, pack_dir=None):
    loader = ContentPackLoader()
    artifacts = None

    if pack_dir:
        artifacts_dir = loader.get_content_from_pack(pack_dir, artifact_type)
        pack_name = os.path.basename(os.path.normpath(pack_dir))
        artifacts = {pack_name: artifacts_dir}
    else:
        packs_dirs = content_utils.get_packs_base_paths()
        artifacts = loader.get_content(packs_dirs, artifact_type)

    artifacts_dict = {}
    for pack_name, pack_path in artifacts.items():
        artifacts_paths = registrar.get_resources_from_pack(pack_path)
        for artifact_path in artifacts_paths:
            artifact = meta_loader.load(artifact_path)
            if artifact_type == "sensors":
                sensors_dir = os.path.dirname(artifact_path)
                sensor_file_path = os.path.join(sensors_dir, artifact["entry_point"])
                artifact["artifact_uri"] = "file://" + sensor_file_path
            name = artifact.get("name", None) or artifact.get("class_name", None)
            if not artifact.get("pack", None):
                artifact["pack"] = pack_name
            ref = ResourceReference.to_string_reference(name=name, pack=pack_name)
            API_MODEL = API_MODELS_ARTIFACT_TYPES[artifact_type]
            # Following conversions are required because we add some fields with
            # default values in db model. If we don't do these conversions,
            # we'll see a unnecessary diff for those fields.
            artifact_api = API_MODEL(**artifact)
            artifact_db = API_MODEL.to_model(artifact_api)
            artifact_api = API_MODEL.from_model(artifact_db)
            artifacts_dict[ref] = artifact_api

    return artifacts_dict

    def test_register_all_configs_with_config_schema_validation_validation_failure_1(self):
        # Verify DB is empty
        pack_dbs = Pack.get_all()
        config_dbs = Config.get_all()

        self.assertEqual(len(pack_dbs), 0)
        self.assertEqual(len(config_dbs), 0)

        registrar = ConfigsRegistrar(use_pack_cache=False, fail_on_failure=True,
                                     validate_configs=True)
        registrar._pack_loader.get_packs = mock.Mock()
        registrar._pack_loader.get_packs.return_value = {'dummy_pack_6': PACK_6_PATH}

        # Register ConfigSchema for pack
        registrar._register_pack_db = mock.Mock()
        registrar._register_pack(pack_name='dummy_pack_5', pack_dir=PACK_6_PATH)
        packs_base_paths = content_utils.get_packs_base_paths()

        if six.PY3:
            expected_msg = ('Failed validating attribute "regions" in config for pack '
                            '"dummy_pack_6" (.*?): 1000 is not of type \'array\'')
        else:
            expected_msg = ('Failed validating attribute "regions" in config for pack '
                            '"dummy_pack_6" (.*?): 1000 is not of type u\'array\'')

        self.assertRaisesRegexp(ValueError, expected_msg,
                                registrar.register_from_packs,
                                base_dirs=packs_base_paths)

    def collect_pack_content(output_path):
        """
        Copy pack contents to the output path.

        :param output_path: Path where pack contents will be copied to.
        :type output_path: ``str``
        """
        LOG.debug('Including content')

        packs_base_paths = get_packs_base_paths()
        for index, packs_base_path in enumerate(packs_base_paths, 1):
            dst = os.path.join(output_path, 'dir-%s' % index)

            try:
                shutil.copytree(src=packs_base_path, dst=dst)
            except IOError:
                continue

        base_pack_dirs = get_dirs_in_path(file_path=output_path)

        for base_pack_dir in base_pack_dirs:
            pack_dirs = get_dirs_in_path(file_path=base_pack_dir)

            for pack_dir in pack_dirs:
                process_content_pack_dir(pack_dir=pack_dir)

    def test_register_packs(self):
        # Verify DB is empty
        pack_dbs = Pack.get_all()
        config_schema_dbs = ConfigSchema.get_all()

        self.assertEqual(len(pack_dbs), 0)
        self.assertEqual(len(config_schema_dbs), 0)

        registrar = ResourceRegistrar(use_pack_cache=False)
        registrar._pack_loader.get_packs = mock.Mock()
        registrar._pack_loader.get_packs.return_value = {'dummy_pack_1': PACK_PATH_1}
        packs_base_paths = content_utils.get_packs_base_paths()
        registrar.register_packs(base_dirs=packs_base_paths)

        # Verify pack and schema have been registered
        pack_dbs = Pack.get_all()
        config_schema_dbs = ConfigSchema.get_all()

        self.assertEqual(len(pack_dbs), 1)
        self.assertEqual(len(config_schema_dbs), 1)

        self.assertEqual(pack_dbs[0].name, 'dummy_pack_1')
        self.assertEqual(len(pack_dbs[0].contributors), 2)
        self.assertEqual(pack_dbs[0].contributors[0], 'John Doe1 <[email protected]>')
        self.assertEqual(pack_dbs[0].contributors[1], 'John Doe2 <[email protected]>')
        self.assertTrue('api_key' in config_schema_dbs[0].attributes)
        self.assertTrue('api_secret' in config_schema_dbs[0].attributes)

    def test_register_configs_for_all_packs(self):
        # Verify DB is empty
        pack_dbs = Pack.get_all()
        config_dbs = Config.get_all()

        self.assertEqual(len(pack_dbs), 0)
        self.assertEqual(len(config_dbs), 0)

        registrar = ConfigsRegistrar(use_pack_cache=False)
        registrar._pack_loader.get_packs = mock.Mock()
        registrar._pack_loader.get_packs.return_value = {"dummy_pack_1": PACK_PATH}
        packs_base_paths = content_utils.get_packs_base_paths()
        registrar.register_configs_for_all_packs(base_dirs=packs_base_paths)

        # Verify pack and schema have been registered
        pack_dbs = Pack.get_all()
        config_dbs = Config.get_all()

        self.assertEqual(len(pack_dbs), 1)
        self.assertEqual(len(config_dbs), 1)

        config_db = config_dbs[0]
        self.assertEqual(config_db.values["api_key"], "{{user.api_key}}")
        self.assertEqual(config_db.values["api_secret"], "{{user.api_secret}}")
        self.assertEqual(config_db.values["region"], "us-west-1")

    def test_register_pack_pack_ref(self):
        # Verify DB is empty
        pack_dbs = Pack.get_all()

        self.assertEqual(len(pack_dbs), 0)

        registrar = ResourceRegistrar(use_pack_cache=False)
        registrar._pack_loader.get_packs = mock.Mock()
        registrar._pack_loader.get_packs.return_value = {
            'dummy_pack_1': PACK_PATH_1,
            'dummy_pack_6': PACK_PATH_6
        }
        packs_base_paths = content_utils.get_packs_base_paths()
        registrar.register_packs(base_dirs=packs_base_paths)

        # Ref is provided
        pack_db = Pack.get_by_name('dummy_pack_6')
        self.assertEqual(pack_db.ref, 'dummy_pack_6_ref')
        self.assertEqual(len(pack_db.contributors), 0)

        # Ref is not provided, directory name should be used
        pack_db = Pack.get_by_name('dummy_pack_1')
        self.assertEqual(pack_db.ref, 'dummy_pack_1')

        # "ref" is not provided, but "name" is
        registrar._register_pack_db(pack_name=None, pack_dir=PACK_PATH_7)

        pack_db = Pack.get_by_name('dummy_pack_7_name')
        self.assertEqual(pack_db.ref, 'dummy_pack_7_name')

        # "ref" is not provided and "name" contains invalid characters
        expected_msg = 'contains invalid characters'
        self.assertRaisesRegexp(ValueError, expected_msg, registrar._register_pack_db,
                                pack_name=None, pack_dir=PACK_PATH_8)

    def _setup_pack_virtualenv(self, pack_name, update=False):
        """
        Setup virtual environment for the provided pack.

        :param pack_name: Pack name.
        :type pack_name: ``str``
        """
        # Prevent directory traversal by whitelisting allowed characters in the
        # pack name
        if not re.match(PACK_NAME_WHITELIST, pack_name):
            raise ValueError('Invalid pack name "%s"' % (pack_name))

        self.logger.debug('Setting up virtualenv for pack "%s"' % (pack_name))

        virtualenv_path = os.path.join(self._base_virtualenvs_path, quote_unix(pack_name))

        # Ensure pack directory exists in one of the search paths
        pack_path = get_pack_directory(pack_name=pack_name)

        if not pack_path:
            packs_base_paths = get_packs_base_paths()
            search_paths = ', '.join(packs_base_paths)
            msg = 'Pack "%s" is not installed. Looked in: %s' % (pack_name, search_paths)
            raise Exception(msg)

        if not os.path.exists(self._base_virtualenvs_path):
            os.makedirs(self._base_virtualenvs_path)

        # If we don't want to update, or if the virtualenv doesn't exist, let's create it.
        if not update or not os.path.exists(virtualenv_path):
            # 0. Delete virtual environment if it exists
            self._remove_virtualenv(virtualenv_path=virtualenv_path)

            # 1. Create virtual environment
            self.logger.debug('Creating virtualenv for pack "%s" in "%s"' %
                              (pack_name, virtualenv_path))
            self._create_virtualenv(virtualenv_path=virtualenv_path)

        # 2. Install base requirements which are common to all the packs
        self.logger.debug('Installing base requirements')
        for requirement in BASE_PACK_REQUIREMENTS:
            self._install_requirement(virtualenv_path=virtualenv_path,
                                      requirement=requirement)

        # 3. Install pack-specific requirements
        requirements_file_path = os.path.join(pack_path, 'requirements.txt')
        has_requirements = os.path.isfile(requirements_file_path)

        if has_requirements:
            self.logger.debug('Installing pack specific requirements from "%s"' %
                              (requirements_file_path))
            self._install_requirements(virtualenv_path, requirements_file_path)
        else:
            self.logger.debug('No pack specific requirements found')

        self.logger.debug('Virtualenv for pack "%s" successfully %s in "%s"' %
                          (pack_name,
                           'updated' if update else 'created',
                           virtualenv_path))

    def run(self, pack):
        """
        :param pack: Installed Pack Name to get info about
        :type pack: ``str``
        """
        packs_base_paths = get_packs_base_paths()

        pack_path = None
        metadata_file = None
        for packs_base_path in packs_base_paths:
            pack_path = os.path.join(packs_base_path, pack)
            pack_yaml_path = os.path.join(pack_path, MANIFEST_FILE_NAME)

            if os.path.isfile(pack_yaml_path):
                metadata_file = pack_yaml_path
                break

        # Pack doesn't exist, finish execution normally with empty metadata
        if not os.path.isdir(pack_path):
            return {
                'pack': None,
                'git_status': None
            }

        if not metadata_file:
            error = ('Pack "%s" doesn\'t contain pack.yaml file.' % (pack))
            raise Exception(error)

        try:
            details = self._parse_yaml_file(metadata_file)
        except Exception as e:
            error = ('Pack "%s" doesn\'t contain a valid pack.yaml file: %s' % (pack, str(e)))
            raise Exception(error)

        try:
            repo = Repo(pack_path)
            git_status = "Status:\n%s\n\nRemotes:\n%s" % (
                repo.git.status().split('\n')[0],
                "\n".join([remote.url for remote in repo.remotes])
            )

            ahead_behind = repo.git.rev_list(
                '--left-right', '--count', 'HEAD...origin/master'
            ).split()
            # Dear god.
            if ahead_behind != [u'0', u'0']:
                git_status += "\n\n"
                git_status += "%s commits ahead " if ahead_behind[0] != u'0' else ""
                git_status += "and " if u'0' not in ahead_behind else ""
                git_status += "%s commits behind " if ahead_behind[1] != u'0' else ""
                git_status += "origin/master."
        except InvalidGitRepositoryError:
            git_status = None

        return {
            'pack': details,
            'git_status': git_status
        }

    def test_register_pack_invalid_config_schema_invalid_attribute(self):
        registrar = ResourceRegistrar(use_pack_cache=False, fail_on_failure=True)
        registrar._pack_loader.get_packs = mock.Mock()
        registrar._pack_loader.get_packs.return_value = {'dummy_pack_18': PACK_PATH_18}
        packs_base_paths = content_utils.get_packs_base_paths()

        expected_msg = r'Additional properties are not allowed \(\'invalid\' was unexpected\)'
        self.assertRaisesRegexp(ValueError, expected_msg, registrar.register_packs,
                                base_dirs=packs_base_paths)

    def test_register_pack_invalid_python_versions_attribute(self):
        registrar = ResourceRegistrar(use_pack_cache=False, fail_on_failure=True)
        registrar._pack_loader.get_packs = mock.Mock()
        registrar._pack_loader.get_packs.return_value = {'dummy_pack_21': PACK_PATH_21}
        packs_base_paths = content_utils.get_packs_base_paths()

        expected_msg = r"'4' is not one of \['2', '3'\]"
        self.assertRaisesRegexp(ValueError, expected_msg, registrar.register_packs,
                                base_dirs=packs_base_paths)