Python parser.JobParser类代码示例

 def create_bbb_job(self, filename, output_dir="/tmp/"):  # pylint: disable=no-self-use
     device = NewDevice(os.path.join(os.path.dirname(__file__), "../devices/bbb-01.yaml"))
     kvm_yaml = os.path.join(os.path.dirname(__file__), filename)
     with open(kvm_yaml) as sample_job_data:
         parser = JobParser()
         job = parser.parse(sample_job_data, device, 4212, None, output_dir=output_dir)
     return job

 def test_job_no_tags(self):
     with open(self.filename) as yaml_data:
         alpha_data = yaml.load(yaml_data)
     for vlan_key, vlan_value in alpha_data['protocols'][VlandProtocol.name].items():
         alpha_data['protocols'][VlandProtocol.name][vlan_key] = {'tags': []}
     # removed tags from original job to simulate job where any interface tags will be acceptable
     self.assertEqual(
         alpha_data['protocols'][VlandProtocol.name],
         {'vlan_one': {'tags': []}}
     )
     parser = JobParser()
     job = parser.parse(yaml.dump(alpha_data), self.device, 4212, None, None, None, output_dir='/tmp/')
     job.validate()
     vprotocol = [vprotocol for vprotocol in job.protocols if vprotocol.name == VlandProtocol.name][0]
     self.assertTrue(vprotocol.valid)
     self.assertEqual(vprotocol.names, {'vlan_one': '4212vlanone'})
     self.assertFalse(vprotocol.check_timeout(120, {'request': 'no call'}))
     self.assertRaises(JobError, vprotocol.check_timeout, 60, 'deploy_vlans')
     self.assertRaises(JobError, vprotocol.check_timeout, 60, {'request': 'deploy_vlans'})
     self.assertTrue(vprotocol.check_timeout(120, {'request': 'deploy_vlans'}))
     for vlan_name in job.parameters['protocols'][VlandProtocol.name]:
         if vlan_name == 'yaml_line':
             continue
         self.assertIn(vlan_name, vprotocol.params)
         self.assertIn('switch', vprotocol.params[vlan_name])
         self.assertIn('port', vprotocol.params[vlan_name])

 def test_compatibility(self):
     factory = Factory()
     job = factory.create_kvm_job('sample_jobs/kvm.yaml', mkdtemp())
     pipe = job.describe()
     self.assertEqual(pipe['compatibility'], DeployImages.compatibility)
     self.assertEqual(job.compatibility, DeployImages.compatibility)
     kvm_yaml = os.path.join(os.path.dirname(__file__), 'sample_jobs/kvm.yaml')
     job_def = yaml.load(open(kvm_yaml, 'r'))
     job_def['compatibility'] = job.compatibility
     parser = JobParser()
     device = NewDevice(os.path.join(os.path.dirname(__file__), '../devices/kvm01.yaml'))
     try:
         job = parser.parse(yaml.dump(job_def), device, 4212, None, output_dir=mkdtemp())
     except NotImplementedError:
         # some deployments listed in basics.yaml are not implemented yet
         pass
     self.assertIsNotNone(job)
     job_def['compatibility'] = job.compatibility + 1
     self.assertRaises(
         JobError, parser.parse, yaml.dump(job_def), device, 4212, None, mkdtemp()
     )
     job_def['compatibility'] = 0
     try:
         job = parser.parse(yaml.dump(job_def), device, 4212, None, output_dir=mkdtemp())
     except NotImplementedError:
         # some deployments listed in basics.yaml are not implemented yet
         pass
     self.assertIsNotNone(job)

    def test_prompt_from_job(self):  # pylint: disable=too-many-locals
        """
        Support setting the prompt after login via the job

        Loads a known YAML, adds a prompt to the dict and re-parses the job.
        Checks that the prompt is available in the expect_shell_connection action.
        """
        factory = Factory()
        job = factory.create_bbb_job('sample_jobs/uboot.yaml')
        job.validate()
        uboot = [action for action in job.pipeline.actions if action.name == 'uboot-action'][0]
        retry = [action for action in uboot.internal_pipeline.actions
                 if action.name == 'uboot-retry'][0]
        expect = [action for action in retry.internal_pipeline.actions
                  if action.name == 'expect-shell-connection'][0]
        check = expect.parameters
        device = NewDevice(os.path.join(os.path.dirname(__file__), '../devices/bbb-01.yaml'))
        extra_yaml = os.path.join(os.path.dirname(__file__), 'sample_jobs/uboot.yaml')
        with open(extra_yaml) as data:
            sample_job_string = data.read()
        parser = JobParser()
        sample_job_data = yaml.load(sample_job_string)
        boot = [item['boot'] for item in sample_job_data['actions'] if 'boot' in item][0]
        self.assertIsNotNone(boot)
        sample_job_string = yaml.dump(sample_job_data)
        job = parser.parse(sample_job_string, device, 4212, None, output_dir='/tmp')
        job.validate()
        uboot = [action for action in job.pipeline.actions if action.name == 'uboot-action'][0]
        retry = [action for action in uboot.internal_pipeline.actions
                 if action.name == 'uboot-retry'][0]
        expect = [action for action in retry.internal_pipeline.actions
                  if action.name == 'expect-shell-connection'][0]
        self.assertNotEqual(check, expect.parameters)

 def create_fastboot_job(self, filename, output_dir='/tmp/'):  # pylint: disable=no-self-use
     device = NewDevice(os.path.join(os.path.dirname(__file__), '../devices/nexus4-01.yaml'))
     fastboot_yaml = os.path.join(os.path.dirname(__file__), filename)
     with open(fastboot_yaml) as sample_job_data:
         parser = JobParser()
         job = parser.parse(sample_job_data, device, 4212, None, output_dir=output_dir)
     return job

    def test_device_environment(self):
        data = """
# YAML syntax.
overrides:
 DEBEMAIL: "[email protected]"
 DEBFULLNAME: "Neil Williams"
        """
        job_parser = JobParser()
        device = NewDevice(os.path.join(os.path.dirname(__file__), '../devices/bbb-01.yaml'))
        sample_job_file = os.path.join(os.path.dirname(__file__), 'sample_jobs/uboot-ramdisk.yaml')
        with open(sample_job_file) as sample_job_data:
            job = job_parser.parse(
                sample_job_data, device, 4212, None,
                output_dir='/tmp', env_dut=data)
        self.assertEqual(
            job.parameters['env_dut'],
            data
        )
        job.validate()
        boot_actions = [
            action.internal_pipeline.actions for action in job.pipeline.actions if action.name == 'uboot-action'][0]
        retry = [action for action in boot_actions if action.name == 'uboot-retry'][0]
        boot_env = [action for action in retry.internal_pipeline.actions if action.name == 'export-device-env'][0]
        found = False
        for line in boot_env.env:
            if 'DEBFULLNAME' in line:
                found = True
                # assert that the string containing a space still contains that space and is quoted
                self.assertIn('\\\'Neil Williams\\\'', line)
        self.assertTrue(found)

    def test_compatibility(self):
        """
        Test compatibility support.

        The class to use in the comparison will change according to which class
        is related to the change which caused the compatibility to be modified.
        """
        factory = Factory()
        job = factory.create_kvm_job('sample_jobs/kvm.yaml', mkdtemp())
        pipe = job.describe()
        self.assertEqual(pipe['compatibility'], ExpectShellSession.compatibility)
        self.assertEqual(job.compatibility, ExpectShellSession.compatibility)
        kvm_yaml = os.path.join(os.path.dirname(__file__), 'sample_jobs/kvm.yaml')
        job_def = yaml.load(open(kvm_yaml, 'r'))
        job_def['compatibility'] = job.compatibility
        parser = JobParser()
        device = NewDevice(os.path.join(os.path.dirname(__file__), '../devices/kvm01.yaml'))
        try:
            job = parser.parse(yaml.dump(job_def), device, 4212, None, output_dir=mkdtemp())
        except NotImplementedError:
            # some deployments listed in basics.yaml are not implemented yet
            pass
        self.assertIsNotNone(job)
        job_def['compatibility'] = job.compatibility + 1
        self.assertRaises(
            JobError, parser.parse, yaml.dump(job_def), device, 4212, None, mkdtemp()
        )
        job_def['compatibility'] = 0
        try:
            job = parser.parse(yaml.dump(job_def), device, 4212, None, output_dir=mkdtemp())
        except NotImplementedError:
            # some deployments listed in basics.yaml are not implemented yet
            pass
        self.assertIsNotNone(job)

 def test_job(self):
     user = self.factory.make_user()
     job = TestJob.from_yaml_and_user(
         self.factory.make_job_yaml(), user)
     job_def = yaml.load(job.definition)
     job_ctx = job_def.get('context', {})
     device = Device.objects.get(hostname='fakeqemu1')
     device_config = device.load_device_configuration(job_ctx)  # raw dict
     parser = JobParser()
     obj = PipelineDevice(device_config, device.hostname)
     pipeline_job = parser.parse(job.definition, obj, job.id, None, output_dir='/tmp')
     pipeline_job.pipeline.validate_actions()
     pipeline = pipeline_job.describe()
     map_metadata(yaml.dump(pipeline), job)
     self.assertEqual(MetaType.objects.filter(metatype=MetaType.DEPLOY_TYPE).count(), 1)
     self.assertEqual(MetaType.objects.filter(metatype=MetaType.BOOT_TYPE).count(), 1)
     count = ActionData.objects.all().count()
     self.assertEqual(TestData.objects.all().count(), 1)
     testdata = TestData.objects.all()[0]
     self.assertEqual(testdata.testjob, job)
     for actionlevel in ActionData.objects.all():
         self.assertEqual(actionlevel.testdata, testdata)
     action_levels = []
     for testdata in job.test_data.all():
         action_levels.extend(testdata.actionlevels.all())
     self.assertEqual(count, len(action_levels))
     count = ActionData.objects.filter(meta_type__metatype=MetaType.DEPLOY_TYPE).count()
     self.assertNotEqual(ActionData.objects.filter(meta_type__metatype=MetaType.BOOT_TYPE).count(), 0)
     self.assertEqual(ActionData.objects.filter(meta_type__metatype=MetaType.UNKNOWN_TYPE).count(), 0)
     for actionlevel in ActionData.objects.filter(meta_type__metatype=MetaType.BOOT_TYPE):
         self.assertEqual(actionlevel.testdata.testjob.id, job.id)
     self.assertEqual(ActionData.objects.filter(
         meta_type__metatype=MetaType.DEPLOY_TYPE,
         testdata__testjob=job
     ).count(), count)

    def test_job_parameters(self):
        """
        Test that the job parameters match expected structure
        """
        self.maxDiff = None  # pylint: disable=invalid-name
        job_parser = JobParser()
        cubie = NewDevice(os.path.join(os.path.dirname(__file__), '../devices/cubie1.yaml'))
        sample_job_file = os.path.join(os.path.dirname(__file__), 'sample_jobs/cubietruck-removable.yaml')
        with open(sample_job_file) as sample_job_data:
            job = job_parser.parse(sample_job_data, cubie, 4212, None, None, None, output_dir='/tmp/')
        try:
            job.validate()
        except JobError:
            self.fail(job.pipeline.errors)
        sample_job_data.close()
        description_ref = pipeline_reference('cubietruck-removable.yaml')
        self.assertEqual(description_ref, job.pipeline.describe(False))

        mass_storage = None  # deploy
        for action in job.pipeline.actions:
            if isinstance(action, DeployAction):
                if isinstance(action, MassStorage):
                    self.assertTrue(action.valid)
                    agent = action.parameters['download']['tool']
                    self.assertTrue(agent.startswith('/'))  # needs to be a full path but on the device, so avoid os.path
                    self.assertIn(action.parameters['device'], job.device['parameters']['media']['usb'])
                    mass_storage = action
        self.assertIsNotNone(mass_storage)
        self.assertIn('device', mass_storage.parameters)
        self.assertIn(mass_storage.parameters['device'], cubie['parameters']['media']['usb'])
        self.assertIsNotNone(mass_storage.get_common_data('u-boot', 'device'))
        u_boot_params = cubie['actions']['boot']['methods']['u-boot']
        self.assertEqual(mass_storage.get_common_data('bootloader_prompt', 'prompt'), u_boot_params['parameters']['bootloader_prompt'])

 def test_secondary_media(self):
     """
     Test UBootSecondaryMedia validation
     """
     job_parser = JobParser()
     cubie = NewDevice(os.path.join(os.path.dirname(__file__), '../devices/cubie1.yaml'))
     sample_job_file = os.path.join(os.path.dirname(__file__), 'sample_jobs/cubietruck-removable.yaml')
     sample_job_data = open(sample_job_file)
     job = job_parser.parse(sample_job_data, cubie, 4212, None, None, None,
                            output_dir='/tmp/')
     job.validate()
     sample_job_data.close()
     u_boot_media = [action for action in job.pipeline.actions if action.name == 'uboot-action'][1].internal_pipeline.actions[0]
     self.assertIsInstance(u_boot_media, UBootSecondaryMedia)
     self.assertEqual([], u_boot_media.errors)
     self.assertEqual(u_boot_media.parameters['kernel'], '/boot/vmlinuz-3.16.0-4-armmp-lpae')
     self.assertEqual(u_boot_media.parameters['kernel'], u_boot_media.get_common_data('file', 'kernel'))
     self.assertEqual(u_boot_media.parameters['ramdisk'], u_boot_media.get_common_data('file', 'ramdisk'))
     self.assertEqual(u_boot_media.parameters['dtb'], u_boot_media.get_common_data('file', 'dtb'))
     self.assertEqual(u_boot_media.parameters['root_uuid'], u_boot_media.get_common_data('uuid', 'root'))
     part_reference = '%s:%s' % (
         job.device['parameters']['media']['usb'][u_boot_media.get_common_data('u-boot', 'device')]['device_id'],
         u_boot_media.parameters['boot_part']
     )
     self.assertEqual(part_reference, u_boot_media.get_common_data('uuid', 'boot_part'))
     self.assertEqual(part_reference, "0:1")

 def test_extra_options(self):
     device = NewDevice(os.path.join(os.path.dirname(__file__), '../devices/kvm01.yaml'))
     kvm_yaml = os.path.join(os.path.dirname(__file__), 'sample_jobs/kvm-inline.yaml')
     with open(kvm_yaml) as sample_job_data:
         job_data = yaml.load(sample_job_data)
     device['actions']['boot']['methods']['qemu']['parameters']['extra'] = yaml.load("""
               - -smp
               - 1
               - -global
               - virtio-blk-device.scsi=off
               - -device virtio-scsi-device,id=scsi
               - --append "console=ttyAMA0 root=/dev/vda rw"
               """)
     self.assertIsInstance(device['actions']['boot']['methods']['qemu']['parameters']['extra'][1], int)
     parser = JobParser()
     job = parser.parse(yaml.dump(job_data), device, 4212, None, None, None,
                        output_dir='/tmp/')
     job.validate()
     boot_image = [action for action in job.pipeline.actions if action.name == 'boot_image_retry'][0]
     boot_qemu = [action for action in boot_image.internal_pipeline.actions if action.name == 'boot_qemu_image'][0]
     qemu = [action for action in boot_qemu.internal_pipeline.actions if action.name == 'execute-qemu'][0]
     self.assertIsInstance(qemu.sub_command, list)
     [self.assertIsInstance(item, str) for item in qemu.sub_command]
     self.assertIn('virtio-blk-device.scsi=off', qemu.sub_command)
     self.assertIn('1', qemu.sub_command)
     self.assertNotIn(1, qemu.sub_command)

 def test_parameter_support(self):  # pylint: disable=too-many-locals
     data = self.factory.make_job_data()
     test_block = [block for block in data['actions'] if 'test' in block][0]
     smoke = test_block['test']['definitions'][0]
     smoke['parameters'] = {
         'VARIABLE_NAME_1': "first variable value",
         'VARIABLE_NAME_2': "second value"
     }
     job = TestJob.from_yaml_and_user(yaml.dump(data), self.user)
     job_def = yaml.load(job.definition)
     job_ctx = job_def.get('context', {})
     device = Device.objects.get(hostname='fakeqemu1')
     device_config = device.load_device_configuration(job_ctx, system=False)  # raw dict
     parser = JobParser()
     obj = PipelineDevice(device_config, device.hostname)
     pipeline_job = parser.parse(job.definition, obj, job.id, None, "", output_dir='/tmp')
     allow_missing_path(pipeline_job.pipeline.validate_actions, self,
                        'qemu-system-x86_64')
     pipeline = pipeline_job.describe()
     device_values = _get_device_metadata(pipeline['device'])
     try:
         testdata, _ = TestData.objects.get_or_create(testjob=job)
     except (MultipleObjectsReturned):
         self.fail('multiple objects')
     for key, value in device_values.items():
         if not key or not value:
             continue
         testdata.attributes.create(name=key, value=value)
     retval = _get_job_metadata(pipeline['job']['actions'])
     self.assertIn('test.0.common.definition.parameters.VARIABLE_NAME_2', retval)
     self.assertIn('test.0.common.definition.parameters.VARIABLE_NAME_1', retval)
     self.assertEqual(retval['test.0.common.definition.parameters.VARIABLE_NAME_1'], 'first variable value')
     self.assertEqual(retval['test.0.common.definition.parameters.VARIABLE_NAME_2'], 'second value')

 def create_ssh_job(self, filename, output_dir=None):  # pylint: disable=no-self-use
     device = NewDevice(os.path.join(os.path.dirname(__file__), '../devices/ssh-host-01.yaml'))
     kvm_yaml = os.path.join(os.path.dirname(__file__), filename)
     with open(kvm_yaml) as sample_job_data:
         parser = JobParser()
         job = parser.parse(sample_job_data, device, 0, socket_addr=None, output_dir=output_dir)
     return job

    def parse_job_file(self, filename, oob_file):
        """
        Uses the parsed device_config instead of the old Device class
        so it can fail before the Pipeline is made.
        Avoids loading all configuration for all supported devices for every job.
        """
        if is_pipeline_job(filename):
            # Prepare the pipeline from the file using the parser.
            device = None  # secondary connections do not need a device
            if self.args.target:
                device = NewDevice(self.args.target)  # DeviceParser
            parser = JobParser()
            job = None
            try:
                env_dut = str(open(self.args.env_dut_path, 'r').read())
            except (TypeError, AttributeError):
                env_dut = None

            try:
                with open(filename) as f_in:
                    job = parser.parse(f_in, device, self.args.job_id,
                                       socket_addr=self.args.socket_addr,
                                       output_dir=self.args.output_dir,
                                       env_dut=env_dut)

            except JobError as exc:
                logging.error("Invalid job submission: %s" % exc)
                exit(1)
            # FIXME: NewDevice schema needs a validation parser
            # device.check_config(job)
            return get_pipeline_runner(job), job.parameters

        # everything else is assumed to be JSON
        return run_legacy_job, json.load(open(filename))

 def test_deployment(self):
     job_parser = JobParser()
     cubie = NewDevice(os.path.join(os.path.dirname(__file__), "../devices/cubie1.yaml"))
     sample_job_file = os.path.join(os.path.dirname(__file__), "sample_jobs/cubietruck-removable.yaml")
     sample_job_data = open(sample_job_file)
     job = job_parser.parse(sample_job_data, cubie, 4212, None, output_dir="/tmp/")
     job.validate()
     self.assertIn("usb", cubie["parameters"]["media"].keys())
     deploy_params = [methods for methods in job.parameters["actions"] if "deploy" in methods.keys()][0]["deploy"]
     self.assertIn("device", deploy_params)
     self.assertIn(deploy_params["device"], cubie["parameters"]["media"]["usb"])
     self.assertIn("uuid", cubie["parameters"]["media"]["usb"][deploy_params["device"]])
     self.assertIn("device_id", cubie["parameters"]["media"]["usb"][deploy_params["device"]])
     self.assertNotIn("boot_part", cubie["parameters"]["media"]["usb"][deploy_params["device"]])
     deploy_action = job.pipeline.actions[0]
     self.assertIn("lava_test_results_dir", deploy_action.data)
     self.assertIn("/lava-", deploy_action.data["lava_test_results_dir"])
     self.assertIsInstance(deploy_action, MassStorage)
     self.assertIn("image", deploy_action.parameters.keys())
     dd_action = deploy_action.internal_pipeline.actions[1]
     self.assertEqual(
         dd_action.boot_params[dd_action.parameters["device"]]["uuid"], "usb-SanDisk_Ultra_20060775320F43006019-0:0"
     )
     self.assertEqual("0", "%s" % dd_action.get_common_data("u-boot", "boot_part"))
     self.assertTrue(type(dd_action.get_common_data("uuid", "boot_part")) is str)
     self.assertEqual("0:1", dd_action.get_common_data("uuid", "boot_part"))