Python models.DeviceDictionary类代码示例

 def test_select_device(self):
     self.restart()
     hostname = 'fakeqemu3'
     device_dict = DeviceDictionary(hostname=hostname)
     device_dict.parameters = self.conf
     device_dict.save()
     device = self.factory.make_device(self.device_type, hostname)
     job = TestJob.from_yaml_and_user(
         self.factory.make_job_yaml(),
         self.factory.make_user())
     # this uses the system jinja2 path - local changes to the qemu.jinja2
     # will not be available.
     selected = select_device(job, self.dispatchers)
     self.assertIsNone(selected)
     job.actual_device = device
     selected = select_device(job, self.dispatchers)
     self.assertIsNone(selected)
     device.worker_host = self.worker
     selected = select_device(job, self.dispatchers)
     self.assertIsNone(selected)
     # device needs to be in reserved state
     # fake up the assignment which needs a separate test
     job.actual_device = device
     job.save()
     device.current_job = job
     device.status = Device.RESERVED
     device.save()
     selected = select_device(job, self.dispatchers)
     self.assertEqual(selected, device)

 def test_pipeline_device(self):
     foo = DeviceDictionary(hostname='foo')
     foo.parameters = {
         'bootz': {
             'kernel': '0x4700000',
             'ramdisk': '0x4800000',
             'dtb': '0x4300000'
         },
         'media': {
             'usb': {
                 'UUID-required': True,
                 'SanDisk_Ultra': {
                     'uuid': 'usb-SanDisk_Ultra_20060775320F43006019-0:0',
                     'device_id': 0
                 },
                 'sata': {
                     'UUID-required': False
                 }
             }
         }
     }
     device = PipelineDevice(foo.parameters, 'foo')
     self.assertEqual(device.target, 'foo')
     self.assertIn('power_state', device)
     self.assertEqual(device.power_state, '')  # there is no power_on_command for this device, so the property is ''
     self.assertTrue(hasattr(device, 'power_state'))
     self.assertFalse(hasattr(device, 'hostname'))
     self.assertIn('hostname', device)

    def test_menu_device(self):
        job_ctx = {}
        hostname = 'mustang01'
        device_dict = DeviceDictionary(hostname=hostname)
        device_dict.parameters = self.conf
        device_dict.save()
        device = self.factory.make_device(self.device_type, hostname)
        self.assertEqual(device.device_type.name, 'mustang-uefi')

        device_data = devicedictionary_to_jinja2(
            device_dict.parameters,
            device_dict.parameters['extends']
        )
        template = prepare_jinja_template(hostname, device_data, system_path=False, path=self.jinja_path)
        config_str = template.render(**job_ctx)
        self.assertIsNotNone(config_str)
        config = yaml.load(config_str)
        self.assertIsNotNone(config)
        self.assertEqual(config['device_type'], self.device_type.name)
        self.assertIsNotNone(config['parameters'])
        self.assertIsNotNone(config['actions']['boot']['methods']['uefi-menu']['nfs'])
        menu_data = config['actions']['boot']['methods']['uefi-menu']['nfs']
        tftp_menu = [item for item in menu_data if 'items' in item['select'] and 'TFTP' in item['select']['items'][0]][0]
        tftp_mac = self.conf['tftp_mac']
        # value from device dictionary correctly replaces device type default
        self.assertIn(tftp_mac, tftp_menu['select']['items'][0])

    def test_vland_jinja2(self):
        """
        Test complex device dictionary values

        The reference data can cross lines but cannot be indented as the pprint
        object in utils uses indent=0, width=80 for YAML compatibility.
        The strings read in from config files can have indenting spaces, these
        are removed in the pprint.
        """
        data = """{% extends 'vland.jinja2' %}
{% set interfaces = ['eth0', 'eth1'] %}
{% set sysfs = {'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0',
'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1'} %}
{% set mac_addr = {'eth0': 'f0:de:f1:46:8c:21', 'eth1': '00:24:d7:9b:c0:8c'} %}
{% set tags = {'eth0': ['1G', '10G'], 'eth1': ['1G']} %}
{% set map = {'eth0': {'192.168.0.2': 5}, 'eth1': {'192.168.0.2': 7}} %}
"""
        result = {
            'interfaces': ['eth0', 'eth1'],
            'sysfs': {
                'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1',
                'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0'
            },
            'extends': 'vland.jinja2',
            'mac_addr': {
                'eth1': '00:24:d7:9b:c0:8c',
                'eth0': 'f0:de:f1:46:8c:21'
            },
            'tags': {
                'eth1': ['1G'],
                'eth0': ['1G', '10G']
            },
            'map': {
                'eth0': {
                    '192.168.0.2': 5
                },
                'eth1': {
                    '192.168.0.2': 7
                }
            }
        }
        dictionary = jinja2_to_devicedictionary(data_dict=data)
        self.assertEqual(result, dictionary)
        jinja2_str = devicedictionary_to_jinja2(data_dict=dictionary, extends='vland.jinja2')
        # ordering within the dict can change but each line needs to still appear
        for line in str(data).split('\n'):
            self.assertIn(line, str(jinja2_str))

        # create a DeviceDictionary for this test
        vlan = DeviceDictionary(hostname='vlanned1')
        vlan.parameters = dictionary
        vlan.save()
        del vlan
        vlan = DeviceDictionary.get('vlanned1')
        cmp_str = str(devicedictionary_to_jinja2(vlan.parameters, 'vland.jinja2'))
        for line in str(data).split('\n'):
            self.assertIn(line, cmp_str)

 def test_new_dictionary(self):
     foo = JobPipeline.get('foo')
     self.assertIsNone(foo)
     foo = DeviceDictionary(hostname='foo')
     foo.save()
     self.assertEqual(foo.hostname, 'foo')
     self.assertIsInstance(foo, DeviceDictionary)
     foo = DeviceDictionary.get('foo')
     self.assertIsNotNone(foo)

 def test_make_device(self):
     hostname = 'fakeqemu3'
     device_dict = DeviceDictionary(hostname=hostname)
     device_dict.parameters = self.conf
     device_dict.save()
     device = self.factory.make_device(self.device_type, hostname)
     self.assertEqual(device.device_type.name, 'qemu')
     job = self.factory.make_job_yaml()
     self.assertIsNotNone(job)

    def test_jinja_postgres_loader(self):
        # path used for the device_type template
        jinja2_path = os.path.realpath(os.path.join(__file__, '..', '..', '..', 'etc', 'dispatcher-config'))
        self.assertTrue(os.path.exists(jinja2_path))
        device_type = 'cubietruck'
        # pretend this was already imported into the database and use for comparison later.
        device_dictionary = {
            'usb_label': 'SanDisk_Ultra',
            'sata_label': 'ST160LM003',
            'usb_uuid': "usb-SanDisk_Ultra_20060775320F43006019-0:0",
            'sata_uuid': "ata-ST160LM003_HN-M160MBB_S2SYJ9KC102184",
            'connection_command': 'telnet localhost 6002'
        }

        # create a DeviceDictionary for this test
        cubie = DeviceDictionary(hostname='cubie')
        cubie.parameters = device_dictionary
        cubie.save()

        dict_loader = jinja2.DictLoader(
            {
                'cubie.yaml':
                devicedictionary_to_jinja2(cubie.parameters, '%s.yaml' % device_type)
            }
        )

        type_loader = jinja2.FileSystemLoader([os.path.join(jinja2_path, 'device-types')])
        env = jinja2.Environment(
            loader=jinja2.ChoiceLoader([dict_loader, type_loader]),
            trim_blocks=True)
        template = env.get_template("%s.yaml" % 'cubie')
        device_configuration = template.render()
        yaml_data = yaml.load(device_configuration)
        self.assertIn('timeouts', yaml_data)
        self.assertIn('parameters', yaml_data)
        self.assertIn('bootz', yaml_data['parameters'])
        self.assertIn('media', yaml_data['parameters'])
        self.assertIn('usb', yaml_data['parameters']['media'])
        self.assertIn(device_dictionary['usb_label'], yaml_data['parameters']['media']['usb'])
        self.assertIn('uuid', yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']])
        self.assertEqual(
            yaml_data['parameters']['media']['usb'][device_dictionary['usb_label']]['uuid'],
            device_dictionary['usb_uuid']
        )
        self.assertIn('commands', yaml_data)
        self.assertIn('connect', yaml_data['commands'])
        self.assertEqual(
            device_dictionary['connection_command'],
            yaml_data['commands']['connect'])
        device = PipelineDevice(yaml_data, 'cubie')
        self.assertIn('power_state', device)
        # cubie1 has no power_on_command defined
        self.assertEqual(device.power_state, '')
        self.assertTrue(hasattr(device, 'power_state'))
        self.assertFalse(hasattr(device, 'hostname'))
        self.assertIn('hostname', device)

 def test_vlan_interface(self):  # pylint: disable=too-many-locals
     device_dict = DeviceDictionary.get('bbb-01')
     chk = {
         'hostname': 'bbb-01',
         'parameters': {
             'map': {'eth1': {'192.168.0.2': 7}, 'eth0': {'192.168.0.2': 5}},
             'interfaces': ['eth0', 'eth1'],
             'sysfs': {
                 'eth1': '/sys/devices/pci0000:00/0000:00:1c.1/0000:03:00.0/net/eth1',
                 'eth0': '/sys/devices/pci0000:00/0000:00:19.0/net/eth0'
             },
             'mac_addr': {'eth1': '00:24:d7:9b:c0:8c', 'eth0': 'f0:de:f1:46:8c:21'},
             'tags': {'eth1': ['1G'], 'eth0': ['1G', '10G']}}
     }
     self.assertEqual(chk, device_dict.to_dict())
     submission = yaml.load(open(self.filename, 'r'))
     self.assertIn('protocols', submission)
     self.assertIn('lava-vland', submission['protocols'])
     roles = [role for role, _ in submission['protocols']['lava-vland'].iteritems()]
     params = submission['protocols']['lava-vland']
     vlans = {}
     for role in roles:
         for name, tags in params[role].iteritems():
             vlans[name] = tags
     self.assertIn('vlan_one', vlans)
     self.assertIn('vlan_two', vlans)
     jobs = split_multinode_yaml(submission, 'abcdefghijkl')
     job_roles = {}
     for role in roles:
         self.assertEqual(len(jobs[role]), 1)
         job_roles[role] = jobs[role][0]
     for role in roles:
         self.assertIn('device_type', job_roles[role])
         self.assertIn('protocols', job_roles[role])
         self.assertIn('lava-vland', job_roles[role]['protocols'])
     client_job = job_roles['client']
     server_job = job_roles['server']
     self.assertIn('vlan_one', client_job['protocols']['lava-vland'])
     self.assertIn('10G', client_job['protocols']['lava-vland']['vlan_one']['tags'])
     self.assertIn('vlan_two', server_job['protocols']['lava-vland'])
     self.assertIn('1G', server_job['protocols']['lava-vland']['vlan_two']['tags'])
     client_tags = client_job['protocols']['lava-vland']['vlan_one']
     client_dict = DeviceDictionary.get('bbb-01').to_dict()
     for interface, tags in client_dict['parameters']['tags'].iteritems():
         if any(set(tags).intersection(client_tags)):
             self.assertEqual(interface, 'eth0')
             self.assertEqual(
                 client_dict['parameters']['map'][interface],
                 {'192.168.0.2': 5}
             )
     # find_device_for_job would have a call to match_vlan_interface(device, job.definition) added
     bbb1 = Device.objects.get(hostname='bbb-01')
     self.assertTrue(match_vlan_interface(bbb1, client_job))
     cubie1 = Device.objects.get(hostname='ct-01')
     self.assertTrue(match_vlan_interface(cubie1, server_job))

 def test_host_role(self):
     # need a full job to properly test the multinode YAML split
     hostname = 'fakeqemu3'
     self.factory.make_device(self.device_type, hostname)
     device_dict = DeviceDictionary(hostname=hostname)
     device_dict.parameters = self.conf
     device_dict.save()
     # create a new device to allow the submission to reach the multinode YAML test.
     hostname = 'fakeqemu4'
     self.factory.make_device(self.device_type, hostname)
     data = yaml.load(self.factory.make_job_json())
     data['protocols']['lava-multinode']['roles']['host']['count'] = 2
     self.assertRaises(
         SubmissionException, TestJob.from_yaml_and_user,
         yaml.dump(data), self.factory.make_user())

 def test_menu_context(self):
     job_ctx = {
         'menu_early_printk': '',
         'menu_interrupt_prompt': 'Default boot will start in'
     }
     hostname = self.factory.make_fake_mustang_device()
     device_dict = DeviceDictionary.get(hostname)
     device_data = devicedictionary_to_jinja2(
         device_dict.parameters,
         device_dict.parameters['extends']
     )
     template = prepare_jinja_template(hostname, device_data, system_path=False, path=self.jinja_path)
     config_str = template.render(**job_ctx)
     self.assertIsNotNone(config_str)
     config = yaml.load(config_str)
     self.assertIsNotNone(config['actions']['boot']['methods']['uefi-menu']['nfs'])
     menu_data = config['actions']['boot']['methods']['uefi-menu']
     # assert that menu_interrupt_prompt replaces the default 'The default boot selection will start in'
     self.assertEqual(
         menu_data['parameters']['interrupt_prompt'],
         job_ctx['menu_interrupt_prompt']
     )
     # assert that menu_early_printk replaces the default earlyprintk default
     self.assertEqual(
         [e for e in menu_data['nfs'] if 'enter' in e['select'] and 'new Entry' in e['select']['wait']][0]['select']['enter'],
         'console=ttyS0,115200  debug root=/dev/nfs rw nfsroot={NFS_SERVER_IP}:{NFSROOTFS},tcp,hard,intr ip=dhcp'
     )

 def cleanup(self):
     DeviceType.objects.all().delete()
     # make sure the DB is in a clean state wrt devices and jobs
     Device.objects.all().delete()
     TestJob.objects.all().delete()
     [item.delete() for item in DeviceDictionary.object_list()]
     User.objects.all().delete()

def match_vlan_interface(device, job_def):
    if not isinstance(job_def, dict):
        raise RuntimeError("Invalid vlan interface data")
    if 'protocols' not in job_def or 'lava-vland' not in job_def['protocols']:
        return False
    interfaces = []
    logger = logging.getLogger('dispatcher-master')
    device_dict = DeviceDictionary.get(device.hostname).to_dict()
    if 'tags' not in device_dict['parameters']:
        logger.error("%s has no tags in the device dictionary parameters", device.hostname)
        return False
    for vlan_name in job_def['protocols']['lava-vland']:
        tag_list = job_def['protocols']['lava-vland'][vlan_name]['tags']
        for interface, tags in device_dict['parameters']['tags'].iteritems():
            logger.info(
                "Job requests %s for %s, device %s provides %s for %s",
                tag_list, vlan_name, device.hostname, tags, interface)
            # tags & job tags must equal job tags
            # device therefore must support all job tags, not all job tags available on the device need to be specified
            if set(tags) & set(tag_list) == set(tag_list) and interface not in interfaces:
                logger.info("Matched vlan %s to interface %s on %s", vlan_name, interface, device)
                interfaces.append(interface)
                # matched, do not check any further interfaces of this device for this vlan
                break
    logger.info("Matched: %s" % (len(interfaces) == len(job_def['protocols']['lava-vland'].keys())))
    return len(interfaces) == len(job_def['protocols']['lava-vland'].keys())

 def cleanup(self):  # pylint: disable=no-self-use
     DeviceType.objects.all().delete()
     # make sure the DB is in a clean state wrt devices and jobs
     Device.objects.all().delete()
     TestJob.objects.all().delete()
     [item.delete() for item in DeviceDictionary.object_list()]  # pylint: disable=expression-not-assigned
     User.objects.all().delete()
     Group.objects.all().delete()

 def test_from_json_rejects_exclusive(self):
     panda_type = self.factory.ensure_device_type(name='panda')
     panda_board = self.factory.make_device(device_type=panda_type, hostname='panda01')
     self.assertFalse(panda_board.is_exclusive)
     job = TestJob.from_json_and_user(
         self.factory.make_job_json(device_type='panda'),
         self.factory.make_user())
     self.assertEqual(panda_type, job.requested_device_type)
     device_dict = DeviceDictionary.get(panda_board.hostname)
     self.assertIsNone(device_dict)
     device_dict = DeviceDictionary(hostname=panda_board.hostname)
     device_dict.parameters = {'exclusive': 'True'}
     device_dict.save()
     self.assertTrue(panda_board.is_exclusive)
     self.assertRaises(
         DevicesUnavailableException, _check_exclusivity, [panda_board], pipeline=False
     )

 def test_identify_context(self):
     hostname = "fakebbb"
     mustang_type = self.factory.make_device_type('beaglebone-black')
     # this sets a qemu device dictionary, so replace it
     self.factory.make_device(device_type=mustang_type, hostname=hostname)
     mustang = DeviceDictionary(hostname=hostname)
     mustang.parameters = {
         'extends': 'beaglebone-black.jinja2',
         'base_nfsroot_args': '10.16.56.2:/home/lava/debian/nfs/,tcp,hard,intr',
         'console_device': 'ttyO0',  # takes precedence over the job context as the same var name is used.
     }
     mustang.save()
     mustang_dict = mustang.to_dict()
     device = Device.objects.get(hostname="fakebbb")
     self.assertEqual('beaglebone-black', device.device_type.name)
     self.assertTrue(device.is_pipeline)
     context_overrides = map_context_overrides('base.jinja2', 'beaglebone-black.jinja2', system=False)
     job_ctx = {
         'base_uboot_commands': 'dummy commands',
         'usb_uuid': 'dummy usb uuid',
         'console_device': 'ttyAMA0',
         'usb_device_id': 1111111111111111
     }
     device_config = device.load_device_configuration(job_ctx, system=False)  # raw dict
     self.assertIsNotNone(device_config)
     devicetype_blocks = []
     devicedict_blocks = []
     allowed = []
     for key, _ in job_ctx.items():
         if key in context_overrides:
             if key is not 'extends' and key not in mustang_dict['parameters'].keys():
                 allowed.append(key)
             else:
                 devicedict_blocks.append(key)
         else:
             devicetype_blocks.append(key)
     # only values set in job_ctx are checked
     self.assertEqual(set(allowed), {'usb_device_id', 'usb_uuid'})
     self.assertEqual(set(devicedict_blocks), {'console_device'})
     self.assertEqual(set(devicetype_blocks), {'base_uboot_commands'})
     full_list = allowed_overrides(mustang_dict, system=False)
     for key in allowed:
         self.assertIn(key, full_list)