Python io.DocumentIO类代码示例

 def invoke_sub(self):
     bundle = generate_combined_bundle(self.args.serial,
                                       self.args.result_id)
     try:
         print DocumentIO.dumps(bundle)
     except IOError:
         pass

 def test_load_and_save_does_not_clobber_the_data(self):
     original_text = resource_string(
         'linaro_dashboard_bundle', 'test_documents/' +
         self.filename)
     fmt, doc = DocumentIO.loads(original_text)
     final_text = DocumentIO.dumps(doc,)
     final_text += "\n"  # the original string has newline at the end
     self.assertEqual(final_text, original_text)

    def _bundle_results(self, target, signal_director, testdef_objs):
        """ Pulls the results from the target device and builds a bundle
        """
        results_part = target.deployment_data['lava_test_results_part_attr']
        results_part = getattr(target.config, results_part)
        rdir = self.context.host_result_dir
        parse_err_msg = None

        filesystem_access_failure = True

        try:
            with target.file_system(results_part, target.lava_test_results_dir) as d:
                filesystem_access_failure = False
                err_log = os.path.join(d, 'parse_err.log')
                results_dir = os.path.join(d, 'results')
                bundle = lava_test_shell.get_bundle(results_dir, testdef_objs, err_log)
                parse_err_msg = read_content(err_log, ignore_missing=True)
                if os.path.isfile(err_log):
                    os.unlink(err_log)
                # lava/results must be empty, but we keep a copy named
                # lava/results-XXXXXXXXXX for post-mortem analysis
                timestamp = datetime.now().strftime("%s")
                os.rename(results_dir, results_dir + '-' + timestamp)
                utils.ensure_directory(results_dir)
        except Exception as e:
            if filesystem_access_failure:
                # a failure when accessing the filesystem means the device
                # probably crashed. We use the backup bundle then.
                bundle = self._backup_bundle
                logging.warning(
                    """Error extracting test results from device: %s""" % e)
                logging.warning(
                    """This may mean that the device under test crashed. """
                    """We will use test results parsed from the serial """
                    """output as a backup, but note that some test """
                    """artifacts (such as attachments and """
                    """hardware/software contexts) will not be available""")
            else:
                raise e

        signal_director.postprocess_bundle(bundle)

        (fd, name) = tempfile.mkstemp(
            prefix='lava-test-shell', suffix='.bundle', dir=rdir)
        with os.fdopen(fd, 'w') as f:
            DocumentIO.dump(f, bundle)

        printer = PrettyPrinter(self.context)
        printer.print_results(bundle)

        if parse_err_msg:
            raise GeneralError(parse_err_msg)

def generate_bundle(serial=None, result_id=None, test=None,
                    test_id=None, attachments=[]):
    if result_id is None:
        return {}
    config = get_config()
    adb = ADB(serial)
    resultdir = os.path.join(config.resultsdir_android, result_id)
    if not adb.exists(resultdir):
        raise  Exception("The result (%s) is not existed." % result_id)

    bundle_text = adb.read_file(os.path.join(resultdir, "testdata.json"))
    bundle = DocumentIO.loads(bundle_text)[1]
    test_tmp = None
    if test:
        test_tmp = test
    else:
        test_tmp = TestProvider().load_test(bundle['test_runs'][0]['test_id'],
                                             serial)
    if test_id:
        bundle['test_runs'][0]['test_id'] = test_id
    else:
        attrs = bundle['test_runs'][0].get('attributes')
        if attrs:
            run_options = attrs.get('run_options')
            if run_options:
                test_id = '%s(%s)' % (bundle['test_runs'][0]['test_id'],
                                      run_options)
                bundle['test_runs'][0]['test_id'] = test_id

    test_tmp.parse(result_id)
    stdout_text = adb.read_file(os.path.join(resultdir,
                                  os.path.basename(test_tmp.org_ouput_file)))
    if stdout_text is None:
        stdout_text = ''
    stderr_text = adb.read_file(os.path.join(resultdir, 'stderr.log'))
    if stderr_text is None:
        stderr_text = ''
    bundle['test_runs'][0]["test_results"] = test_tmp.parser.results[
                                                        "test_results"]

    ## following part is used for generating the attachment for normal test
    attachment_bundles = []
    for attachment in test_tmp.attachments:
        data_bundle = attachment.generate_bundle(adb=adb, resultsdir=resultdir)
        if data_bundle:
            attachment_bundles.append(data_bundle)

    bundle['test_runs'][0]["attachments"] = attachment_bundles

    ##following used for the attachment for monkeyrunner test
    for attach in attachments:
        if os.path.exists(attach):
            with open(attach, 'rb') as stream:
                data = stream.read()
            if data:
                bundle['test_runs'][0]["attachments"].append({
                            "pathname": os.path.basename(attach),
                            "mime_type": 'image/png',
                            "content": base64.standard_b64encode(data)})
    return bundle

 def test_evolved_document_is_what_we_expect(self):
     DocumentEvolution.evolve_document(self.doc, one_step=True)
     fmt, evolved_doc = DocumentIO.load(
         resource_stream('linaro_dashboard_bundle',
                         'test_documents/evolution_1.7.1.json'),
         retain_order=False)
     self.assertEqual(self.doc, evolved_doc)

 def _savetestdata(self, analyzer_assigned_uuid, run_options=""):
     config = get_config()
     TIMEFORMAT = '%Y-%m-%dT%H:%M:%SZ'
     bundle = {
         'format': config.bundle_format,
         'test_runs': [
             {
             'analyzer_assigned_uuid': analyzer_assigned_uuid,
             'analyzer_assigned_date':
                     self.runner.starttime.strftime(TIMEFORMAT),
             'time_check_performed': False,
             'attributes':{},
             'test_id': self.testname,
             'test_results':[],
             'attachments':[],
             'hardware_context': hwprofile.get_hardware_context(self.adb),
             'software_context': swprofile.get_software_context(self.adb)
             }
         ]
     }
     if run_options:
         bundle['test_runs'][0]['attributes']['run_options'] = run_options
     self._add_install_options(bundle, config)
     filename_host = os.path.join(config.tempdir_host, 'testdata.json')
     write_file(DocumentIO.dumps(bundle), filename_host)
     filename_target = os.path.join(self.resultsdir, 'testdata.json')
     self.adb.push(filename_host, filename_target)

    def _get_bundles(self, files):
        bundles = []
        errors = []
        for fname in files:
            if os.path.splitext(fname)[1] != ".bundle":
                continue
            content = None
            try:
                with open(fname, 'r') as f:
                    doc = DocumentIO.load(f)[1]
                DocumentEvolution.evolve_document(doc)
                bundles.append(doc)
            except ValueError:
                msg = 'Error adding result bundle %s' % fname
                errors.append(msg)
                logging.exception(msg)
                if content:
                    logging.info('Adding bundle as attachment')
                    attachment = create_attachment(fname, content)
                    self.context.test_data.add_attachments([attachment])
            except KeyboardInterrupt:
                raise KeyboardInterrupt
            except:
                msg = 'Unknown error processing bundle' % fname
                logging.exception(msg)
                errors.append(msg)

        if len(errors) > 0:
            msg = ' '.join(errors)
            raise GatherResultsError(msg, bundles)
        return bundles

    def _get_results_from_host(self):
        bundles = []
        errors = []
        try:
            bundle_list = os.listdir(self.context.host_result_dir)
            for bundle_name in bundle_list:
                bundle = "%s/%s" % (self.context.host_result_dir, bundle_name)
                content = None
                try:
                    with open(bundle) as f:
                        doc = DocumentIO.load(f)[1]
                    DocumentEvolution.evolve_document(doc)
                    bundles.append(doc)
                except ValueError:
                    msg = 'Error adding host result bundle %s' % bundle
                    errors.append(msg)
                    logging.exception(msg)
                    if content:
                        logging.info('Adding bundle as attachment')
                        attachment = create_attachment(bundle, content)
                        self.context.test_data.add_attachments([attachment])
        except:
            msg = 'Error getting all results from host'
            logging.exception(msg)
            raise GatherResultsError(msg, bundles)

        if len(errors) > 0:
            msg = ' '.join(errors)
            raise GatherResultsError(msg, bundles)

        return bundles

 def test_load_document(self):
     # Note: resource_string uses posix-style paths
     # regardless of the actual system paths
     fmt, doc = DocumentIO.load(
         resource_stream('linaro_dashboard_bundle',
                         'test_documents/' + self.filename))
     self.assertIsNot(doc, None)

 def test_loader_uses_decimal_to_parse_numbers(self):
     text = resource_string(
         'linaro_dashboard_bundle',
         'test_documents/dummy_doc_with_numbers.json')
     fmt, doc = DocumentIO.loads(text)
     measurement = doc["test_runs"][0]["test_results"][0]["measurement"]
     self.assertEqual(measurement, Decimal("1.5"))
     self.assertTrue(isinstance(measurement, Decimal))

    def _load_bundle(self, local_pathname):
        """
        Load the bundle from local_pathname.

        There are various problems that can happen here but
        they should all be treated equally, the bundle not
        being used. This also transparently does schema validation
        so the chance of getting wrong data is lower.
        """
        with open(local_pathname, 'rt') as stream:
            format, bundle = DocumentIO.load(stream)
            return format, bundle

 def submit_bundle(self, main_bundle, server, stream, token):
     dashboard = _get_dashboard(server, token)
     json_bundle = DocumentIO.dumps(main_bundle)
     job_name = self.context.job_data.get('job_name', "LAVA Results")
     try:
         result = dashboard.put_ex(json_bundle, job_name, stream)
         print >> self.context.oob_file, 'dashboard-put-result:', result
         self.context.output.write_named_data('result-bundle', result)
         logging.info("Dashboard : %s" % result)
     except xmlrpclib.Fault, err:
         logging.warning("xmlrpclib.Fault occurred")
         logging.warning("Fault code: %d" % err.faultCode)
         logging.warning("Fault string: %s" % err.faultString)
         raise OperationFailed("could not push to dashboard")

    def deserialize(self, s_bundle, prefer_evolution):
        """
        Deserializes specified Bundle.

        :Discussion:
            This method also handles internal transaction handling.
            All operations performed during bundle deserialization are
            _rolled_back_ if anything fails.

            If prefer_evolution is enabled then the document is first evolved
            to the latest known format and only then imported into the
            database. This operation is currently disabled to ensure that all
            old documents are imported exactly as before. Enabling it should
            be quite safe though as it passes all tests.

        :Exceptions raised:
            json_schema_validator.ValidationError
                When the document does not match the appropriate schema.
            linaro_dashboard_bundle.errors.DocumentFormatError
                When the document format is not in the known set of formats.
            ValueError
                When the text does not represent a correct JSON document.
        """
        assert s_bundle.is_deserialized is False
        s_bundle.content.open('rb')
        logger = logging.getLogger(__name__)
        try:
            logger.debug("Loading document")
            fmt, doc = DocumentIO.load(s_bundle.content)
            logger.debug("Document loaded")
            if prefer_evolution:
                logger.debug("Evolving document")
                DocumentEvolution.evolve_document(doc)
                logger.debug("Document evolution complete")
                fmt = doc["format"]
        finally:
            s_bundle.content.close()
        importer = self.IMPORTERS.get(fmt)
        if importer is None:
            raise DocumentFormatError(fmt)
        try:
            logger.debug("Importing document")
            importer().import_document(s_bundle, doc)
            logger.debug("Document import complete")
        except Exception as exc:
            logger.debug("Exception while importing document: %r", exc)
            raise

    def invoke(self):

        if not os.path.exists(self.args.result_file):
            raise  LavaCommandError("The specified result file(%s) "
                                    "does not exist." % self.args.result_file)
        msg = "extract attachment file from result bundle file(%s)" % (
                                                       self.args.result_file)
        self.say_begin(msg)
        badchars = "[^a-zA-Z0-9\._-]"
        with open(self.args.result_file) as stream:
            jobdata = stream.read()
            result_data = DocumentIO.loads(jobdata)[1]
            test_runs = result_data.get('test_runs')
            if not self.args.directory:
                attachment_dir = mkdtemp(prefix='attachments-',
                                         dir=os.path.curdir)
            elif not os.path.exists(self.args.directory):
                os.makedirs(self.args.directory)
                attachment_dir = self.args.directory
            elif not os.path.isdir(self.args.directory):
                raise  LavaCommandError(
                            "The specified path(%s) is not a directory."
                                    % self.args.directory)
            else:
                attachment_dir = self.args.directory

            for test in test_runs:
                test_id = test.get('test_id').replace(" ", "_")
                test_id = re.sub(badchars, "_", test_id)
                target_dir = mkdtemp(prefix='%s' % test_id, dir=attachment_dir)
                print "The test id is: %s" % test_id
                attachments = test.get('attachments', [])
                for attach in attachments:
                    pathname = attach.get('pathname')
                    file_name = os.path.basename(pathname)
                    content_decoded = base64.standard_b64decode(
                                                        attach.get("content"))
                    with open(os.path.join(target_dir, file_name), 'w') as fd:
                        fd.write(content_decoded)
            self.say("All attachment files are put under directory(%s)" %
                     (attachment_dir))
        self.say_end(msg)

 def test_dumper_can_dump_decimals(self):
     doc = {
         "format": "Dashboard Bundle Format 1.0",
         "test_runs": [
             {
                 "test_id": "NOT RELEVANT",
                 "analyzer_assigned_date": "2010-11-14T01:03:06Z",
                 "analyzer_assigned_uuid": "NOT RELEVANT",
                 "time_check_performed": False,
                 "test_results": [
                     {
                         "test_case_id": "NOT RELEVANT",
                         "result": "unknown",
                         "measurement": Decimal("1.5")
                     }
                 ]
             }
         ]
     }
     text = DocumentIO.dumps(doc)
     self.assertIn("1.5", text)