Python dataloader.DataLoader类代码示例

def boilerplate_module(modfile, args, interpreter, check, destfile):
    """ simulate what ansible does with new style modules """

    loader = DataLoader()

    complex_args = {}
    if args.startswith("@"):
        # Argument is a YAML file (JSON is a subset of YAML)
        complex_args = utils_vars.combine_vars(complex_args,
                                               loader.load_from_file(args[1:]))
        args = ''
    elif args.startswith("{"):
        # Argument is a YAML document (not a file)
        complex_args = utils_vars.combine_vars(complex_args, loader.load(args))
        args = ''

    if args:
        parsed_args = parse_kv(args)
        complex_args = utils_vars.combine_vars(complex_args, parsed_args)

    task_vars = {}
    if interpreter:
        if '=' not in interpreter:
            print("interpreter must by in the form of \
                   ansible_python_interpreter=/usr/bin/python")
            sys.exit(1)
        interpreter_type, interpreter_path = interpreter.split('=')
        if not interpreter_type.startswith('ansible_'):
            interpreter_type = 'ansible_%s' % interpreter_type
        if not interpreter_type.endswith('_interpreter'):
            interpreter_type = '%s_interpreter' % interpreter_type
        task_vars[interpreter_type] = interpreter_path

    if check:
        complex_args['_ansible_check_mode'] = True

    modname = os.path.basename(modfile)
    modname = os.path.splitext(modname)[0]
    (module_data, module_style, shebang) = module_common.modify_module(
        modname,
        modfile,
        complex_args,
        task_vars=task_vars
    )

    if module_style == 'new' \
       and 'ZIPLOADER_WRAPPER = True' in module_data:
        module_style = 'ziploader'

    modfile2_path = os.path.expanduser(destfile)
    print("* including generated source,\
           if any, saving to: %s" % modfile2_path)
    if module_style not in ('ziploader', 'old'):
        print("* this may offset any line numbers in tracebacks/debuggers!")
    modfile2 = open(modfile2_path, 'w')
    modfile2.write(module_data)
    modfile2.close()
    modfile = modfile2_path

    return (modfile2_path, modname, module_style)

    def _play_prereqs(options):

        # all needs loader
        loader = DataLoader()

        basedir = getattr(options, 'basedir', False)
        if basedir:
            loader.set_basedir(basedir)

        vault_ids = options.vault_ids
        default_vault_ids = C.DEFAULT_VAULT_IDENTITY_LIST
        vault_ids = default_vault_ids + vault_ids

        vault_secrets = CLI.setup_vault_secrets(loader,
                                                vault_ids=vault_ids,
                                                vault_password_files=options.vault_password_files,
                                                ask_vault_pass=options.ask_vault_pass,
                                                auto_prompt=False)
        loader.set_vault_secrets(vault_secrets)

        # create the inventory, and filter it based on the subset specified (if any)
        inventory = InventoryManager(loader=loader, sources=options.inventory)

        # create the variable manager, which will be shared throughout
        # the code, ensuring a consistent view of global variables
        variable_manager = VariableManager(loader=loader, inventory=inventory)

        # load vars from cli options
        variable_manager.extra_vars = load_extra_vars(loader=loader, options=options)
        variable_manager.options_vars = load_options_vars(options, CLI.version_info(gitinfo=False))

        return loader, inventory, variable_manager

class TestDataLoaderWithVault(unittest.TestCase):

    def setUp(self):
        self._loader = DataLoader()
        self._loader.set_vault_password('ansible')

    def tearDown(self):
        pass

    @patch.multiple(DataLoader, path_exists=lambda s, x: True, is_file=lambda s, x: True)
    def test_parse_from_vault_1_1_file(self):
        vaulted_data = """$ANSIBLE_VAULT;1.1;AES256
33343734386261666161626433386662623039356366656637303939306563376130623138626165
6436333766346533353463636566313332623130383662340a393835656134633665333861393331
37666233346464636263636530626332623035633135363732623332313534306438393366323966
3135306561356164310a343937653834643433343734653137383339323330626437313562306630
3035
"""
        if PY3:
            builtins_name = 'builtins'
        else:
            builtins_name = '__builtin__'

        with patch(builtins_name + '.open', mock_open(read_data=vaulted_data.encode('utf-8'))):
            output = self._loader.load_from_file('dummy_vault.txt')
            self.assertEqual(output, dict(foo='bar'))

class TestDataLoader(unittest.TestCase):

    def setUp(self):
        self._loader = DataLoader()

    def tearDown(self):
        pass

    @patch.object(DataLoader, '_get_file_contents')
    def test_parse_json_from_file(self, mock_def):
        mock_def.return_value = ("""{"a": 1, "b": 2, "c": 3}""", True)
        output = self._loader.load_from_file('dummy_json.txt')
        self.assertEqual(output, dict(a=1,b=2,c=3))

    @patch.object(DataLoader, '_get_file_contents')
    def test_parse_yaml_from_file(self, mock_def):
        mock_def.return_value = ("""
        a: 1
        b: 2
        c: 3
        """, True)
        output = self._loader.load_from_file('dummy_yaml.txt')
        self.assertEqual(output, dict(a=1,b=2,c=3))

    @patch.object(DataLoader, '_get_file_contents')
    def test_parse_fail_from_file(self, mock_def):
        mock_def.return_value = ("""
        TEXT:
            ***
               NOT VALID
        """, True)
        self.assertRaises(AnsibleParserError, self._loader.load_from_file, 'dummy_yaml_bad.txt')

class Runner(object):
    def __init__(self, playbook, display, hosts=None, options={}, passwords={}, vault_pass=None):

        self.options = Options()
        for k, v in options.iteritems():
            setattr(self.options, k, v)

        self.display = display
        self.display.verbosity = self.options.verbosity
        # executor has its own verbosity setting
        playbook_executor.verbosity = self.options.verbosity

        # gets data from YAML/JSON files
        self.loader = DataLoader()
        if vault_pass is not None:
            self.loader.set_vault_password(vault_pass)
        elif 'VAULT_PASS' in os.environ:
            self.loader.set_vault_password(os.environ['VAULT_PASS'])

        # all the variables from all the various places
        self.variable_manager = VariableManager()
        if self.options.python_interpreter is not None:
            self.variable_manager.extra_vars = {
                'ansible_python_interpreter': self.options.python_interpreter
            }

        # set inventory, using most of above objects
        self.inventory = Inventory(
            loader=self.loader, variable_manager=self.variable_manager,
            host_list=hosts)

        if len(self.inventory.list_hosts()) == 0:
            self.display.error("Provided hosts list is empty.")
            sys.exit(1)

        self.inventory.subset(self.options.subset)

        if len(self.inventory.list_hosts()) == 0:
            self.display.error("Specified limit does not match any hosts.")
            sys.exit(1)

        self.variable_manager.set_inventory(self.inventory)

        # setup playbook executor, but don't run until run() called
        self.pbex = playbook_executor.PlaybookExecutor(
            playbooks=[playbook],
            inventory=self.inventory,
            variable_manager=self.variable_manager,
            loader=self.loader,
            options=self.options,
            passwords=passwords)

    def run(self):
        # run playbook and get stats
        self.pbex.run()
        stats = self.pbex._tqm._stats

        return stats

def load_vars(fname):
    if not os.path.isfile(fname):
        return {}

    if HAS_ANSIBLE2:
        loader = DataLoader()
        return loader.load_from_file(file_name=fname)
    else:
        vars = {}
        return ansible.utils.load_vars(fname, vars)

class TaskResult:
    """
    This class is responsible for interpretting the resulting data
    from an executed task, and provides helper methods for determining
    the result of a given task.
    """

    def __init__(self, host, task, return_data):
        self._host = host
        self._task = task
        if isinstance(return_data, dict):
            self._result = return_data.copy()
        else:
            self._result = DataLoader().load(return_data)

    def is_changed(self):
        return self._check_key("changed")

    def is_skipped(self):
        # loop results
        if "results" in self._result and self._task.loop:
            results = self._result["results"]
            # Loop tasks are only considered skipped if all items were skipped.
            # some squashed results (eg, yum) are not dicts and can't be skipped individually
            if results and all(isinstance(res, dict) and res.get("skipped", False) for res in results):
                return True

        # regular tasks and squashed non-dict results
        return self._result.get("skipped", False)

    def is_failed(self):
        if (
            "failed_when_result" in self._result
            or "results" in self._result
            and True in [True for x in self._result["results"] if "failed_when_result" in x]
        ):
            return self._check_key("failed_when_result")
        else:
            return self._check_key("failed") or self._result.get("rc", 0) != 0

    def is_unreachable(self):
        return self._check_key("unreachable")

    def _check_key(self, key):
        if "results" in self._result and self._task.loop:
            flag = False
            for res in self._result.get("results", []):
                if isinstance(res, dict):
                    flag |= res.get(key, False)
            return flag
        else:
            return self._result.get(key, False)

class Runner(object):

    def __init__(self, hostnames, playbook, private_key_file, run_data, become_pass=None,
                 verbosity=0, callback=None, subset_pattern=None):

        self.hostnames = hostnames

        self.playbook = os.path.join(playbooks_dir, playbook)
        self.run_data = run_data

        self.options = Options(subset=subset_pattern, private_key_file=private_key_file, verbosity=verbosity)

        self.display = Display()
        self.display.verbosity = verbosity
        playbook_executor.verbosity = verbosity

        passwords = {'become_pass': None}

        # Gets data from YAML/JSON files
        self.loader = DataLoader()
        self.loader.set_vault_password(os.environ.get('VAULT_PASS'))

        self.variable_manager = VariableManager()
        self.variable_manager.extra_vars = self.run_data

        self.inventory = Inventory(loader=self.loader, variable_manager=self.variable_manager, host_list=self.hostnames)
        self.variable_manager.set_inventory(self.inventory)

        self.pbex = playbook_executor.PlaybookExecutor(
            playbooks=[self.playbook],
            inventory=self.inventory,
            variable_manager=self.variable_manager,
            loader=self.loader,
            options=self.options,
            passwords=passwords)

        if callback:
            self.pbex._tqm._stdout_callback = callback

    def run(self):
        self.pbex.run()
        stats = self.pbex._tqm._stats

        run_success = True
        hosts = sorted(stats.processed.keys())
        for h in hosts:
            t = stats.summarize(h)
            if t['unreachable'] > 0 or t['failures'] > 0:
                run_success = False

        return run_success

class TaskResult:
    '''
    This class is responsible for interpretting the resulting data
    from an executed task, and provides helper methods for determining
    the result of a given task.
    '''

    def __init__(self, host, task, return_data):
        self._host = host
        self._task = task
        if isinstance(return_data, dict):
            self._result = return_data.copy()
        else:
            self._result = DataLoader().load(return_data)

    def is_changed(self):
        return self._check_key('changed')

    def is_skipped(self):
        # loop results
        if 'results' in self._result:
            results = self._result['results']
            # Loop tasks are only considered skipped if all items were skipped.
            # some squashed results (eg, yum) are not dicts and can't be skipped individually
            if results and all(isinstance(res, dict) and res.get('skipped', False) for res in results):
                return True

        # regular tasks and squashed non-dict results
        return self._result.get('skipped', False)

    def is_failed(self):
        if 'failed_when_result' in self._result or \
           'results' in self._result and True in [True for x in self._result['results'] if 'failed_when_result' in x]:
            return self._check_key('failed_when_result')
        else:
            return self._check_key('failed') or self._result.get('rc', 0) != 0

    def is_unreachable(self):
        return self._check_key('unreachable')

    def _check_key(self, key):
        '''get a specific key from the result or it's items'''

        if isinstance(self._result, dict) and key in self._result:
            return self._result.get(key, False)
        else:
            flag = False
            for res in self._result.get('results', []):
                if isinstance(res, dict):
                    flag |= res.get(key, False)
            return flag

    def run(self):

        results = None

        super(InventoryCLI, self).run()

        # Initialize needed objects
        if getattr(self, '_play_prereqs', False):
            self.loader, self.inventory, self.vm = self._play_prereqs(self.options)
        else:
            # fallback to pre 2.4 way of initialzing
            from ansible.vars import VariableManager
            from ansible.inventory import Inventory

            self._new_api = False
            self.loader = DataLoader()
            self.vm = VariableManager()

            # use vault if needed
            if self.options.vault_password_file:
                vault_pass = CLI.read_vault_password_file(self.options.vault_password_file, loader=self.loader)
            elif self.options.ask_vault_pass:
                vault_pass = self.ask_vault_passwords()
            else:
                vault_pass = None

            if vault_pass:
                self.loader.set_vault_password(vault_pass)
                # actually get inventory and vars

            self.inventory = Inventory(loader=self.loader, variable_manager=self.vm, host_list=self.options.inventory)
            self.vm.set_inventory(self.inventory)

        if self.options.host:
            hosts = self.inventory.get_hosts(self.options.host)
            if len(hosts) != 1:
                raise AnsibleOptionsError("You must pass a single valid host to --hosts parameter")

            myvars = self._get_host_variables(host=hosts[0])
            self._remove_internal(myvars)

            # FIXME: should we template first?
            results = self.dump(myvars)

        elif self.options.graph:
            results = self.inventory_graph()
        elif self.options.list:
            top = self._get_group('all')
            if self.options.yaml:
                results = self.yaml_inventory(top)
            else:
                results = self.json_inventory(top)
            results = self.dump(results)

        if results:
            # FIXME: pager?
            display.display(results)
            exit(0)

        exit(1)

 def __init__(self, inventory, options=None):
     self.options = self.update_options(options)
     self.inventory = inventory
     self.loader = DataLoader()
     self.variable_manager = VariableManager(
         loader=self.loader, inventory=self.inventory
     )

 def __init__(self, host, task, return_data):
     self._host = host
     self._task = task
     if isinstance(return_data, dict):
         self._result = return_data.copy()
     else:
         self._result = DataLoader().load(return_data)

    def main(self, path):
        data_dir = self.conf['data_dir']
        
        loader = DataLoader()

        full_path="%s/%s" % (data_dir, path)

        if os.path.isfile("%s.yaml" % full_path):
            ds = loader.load_from_file("%s.yaml" % full_path)
        elif os.path.isfile("%s.yml" % full_path):
            ds = loader.load_from_file("%s.yml" % full_path)
        else:
            ds={}
        if ds is None:
            ds = {}

        return ds

 def __init__(self, inventory, options=None):
     if options:
         self.options = options
     self.inventory = inventory
     self.loader = DataLoader()
     self.variable_manager = VariableManager(
         loader=self.loader, inventory=self.inventory
     )

class TestDataLoader(unittest.TestCase):

    def setUp(self):
        self._loader = DataLoader()

    def tearDown(self):
        pass

    @patch.object(DataLoader, '_get_file_contents')
    def test_parse_json_from_file(self, mock_def):
        mock_def.return_value = (b"""{"a": 1, "b": 2, "c": 3}""", True)
        output = self._loader.load_from_file('dummy_json.txt')
        self.assertEqual(output, dict(a=1,b=2,c=3))

    @patch.object(DataLoader, '_get_file_contents')
    def test_parse_yaml_from_file(self, mock_def):
        mock_def.return_value = (b"""
        a: 1
        b: 2
        c: 3
        """, True)
        output = self._loader.load_from_file('dummy_yaml.txt')
        self.assertEqual(output, dict(a=1,b=2,c=3))

    @patch.object(DataLoader, '_get_file_contents')
    def test_parse_fail_from_file(self, mock_def):
        mock_def.return_value = (b"""
        TEXT:
            ***
               NOT VALID
        """, True)
        self.assertRaises(AnsibleParserError, self._loader.load_from_file, 'dummy_yaml_bad.txt')

    @patch('ansible.errors.AnsibleError._get_error_lines_from_file')
    @patch.object(DataLoader, '_get_file_contents')
    def test_tab_error(self, mock_def, mock_get_error_lines):
        mock_def.return_value = (u"""---\nhosts: localhost\nvars:\n  foo: bar\n\tblip: baz""", True)
        mock_get_error_lines.return_value = ('''\tblip: baz''', '''..foo: bar''')
        with self.assertRaises(AnsibleParserError) as cm:
            self._loader.load_from_file('dummy_yaml_text.txt')
        self.assertIn(yaml_strings.YAML_COMMON_LEADING_TAB_ERROR, str(cm.exception))
        self.assertIn('foo: bar', str(cm.exception))