Python ir_config.IRConfig类代码示例

    def batch_generate_tfidf(cls):
        """Batch calculate TFIDF."""

        from ir_log import IRProgressBar
        from ir_config import IRConfig
        from ir_mongodb_helper import IRCollection
        from ir_document_count import IRDocumentCount
        from ir_term_count import IRTermCount
        # get config
        bug_id_name = IRConfig.get_instance().get('bug_id_name')
        summary_name = IRConfig.get_instance().get('bug_summary_name')
        description_name = IRConfig.get_instance().get('bug_description_name')
        tfidf_algorithm = IRConfig.get_instance().get('tfidf_algorithm')
        # prepare collections
        IRDocumentCount.cache_all_data()
        tfidf_collection = IRCollection(
            'bug_db_name', 'bug_tfidf_collection_name', 'w')
        # batch calculate tfidf
        termcount_iterator = IRTermCount.get_iterator()
        bug_count = termcount_iterator.count()
        def iter_term_count(bug):
            summary_tfidf = cls.calculate_tfidf(bug[summary_name],
                                                summary_name, bug_count, None, tfidf_algorithm)
            description_tfidf = cls.calculate_tfidf(bug[description_name],
                                                    description_name, bug_count, None, tfidf_algorithm)
            tfidf_collection.insert({bug_id_name : bug[bug_id_name],
                                     summary_name : summary_tfidf,
                                     description_name : description_tfidf})
        IRProgressBar.execute_iteration_for_cursor(termcount_iterator,
                                                   iter_term_count, "Calculating TFIDF")
        tfidf_collection.create_index([(bug_id_name, IRCollection.ASCENDING)])
        tfidf_collection.close()

    def test_cluster_sentences(test):
        from ir_log import IRLog
        from ir_config import IRConfig
        from ir_sentence import IRSentence

        IRLog.get_instance().start_log()
        IRConfig.get_instance().load('../data/test/bug_test.cfg')
        bug_id = 10000
        description = 'Version: 12.43\n'\
                      'Distribution: Gnome 12.03\n'\
                      '\n'\
                      'Steps to repreduce:\n'\
                      '1. Open firefox.\n'\
                      '2. Click Option\n'\
                      '3. Open firefox\n'\
                      '\n'\
                      'Additional information:\n'\
                      'This is really crazy when it crashed.'
        sentences = IRSentence.get_sentence_from_description(description, bug_id)
        group_id, selected_id = IRSentence.cluster_sentences(sentences, 3)
        groups = []
        for i in range(3):
            groups.append([])
        index = 0
        for id in group_id:
            groups[id].append(index)
            index += 1
        index = 0
        for group in groups:
            IRLog.get_instance().println('Group %d. Representative: %s' % \
                    (index, sentences[selected_id[index]].get_text()))
            for id in group:
                IRLog.get_instance().println(sentences[id].get_text())
            index += 1

    def get_stacktrace_of_bug(cls, bug_id):
        """Get stacktrace from mongodb.

        Args:
            bug_id: int

        Returns:
            [[str]], [[signature]]
        """
        if cls.__is_cache:
            if bug_id in cls.__cache_stacktrace:
                return cls.__cache_stacktrace[bug_id]
        from ir_config import IRConfig
        from ir_mongodb_helper import IRCollection
        bug_id_name = IRConfig.get_instance().get('bug_id_name')
        stacktrace_name = IRConfig.get_instance().get('bug_stacktrace_name')
        text_collection = IRCollection(
            'bug_db_name', 'bug_text_collection_name', 'r')
        res = text_collection.find({bug_id_name : bug_id})
        stacktrace = []
        if res.count() > 0:
            stacktrace = res[0][stacktrace_name]
        if cls.__is_cache:
            cls.__cache_stacktrace[bug_id] = stacktrace
        return stacktrace

    def get_tfidf_of_bug(cls, bug_id):
        """Get tfidf of a bug.

        Args:
            bug_id: int

        Returns:
            [dict, dict], [TFIDF of summary, TFIDF of description]
        """

        if cls.__is_cache:
            if bug_id in cls.__cache:
                return cls.__cache[bug_id]
        # load from db
        from ir_config import IRConfig
        from ir_mongodb_helper import IRCollection
        bug_id_name = IRConfig.get_instance().get('bug_id_name')
        summary_name = IRConfig.get_instance().get('bug_summary_name')
        description_name = IRConfig.get_instance().get('bug_description_name')
        tfidf_collection = IRCollection(
            'bug_db_name', 'bug_tfidf_collection_name', 'r')
        find_result = tfidf_collection.find({bug_id_name : bug_id})
        summary = {}
        description = {}
        if find_result.count() > 0:
            summary = find_result[0][summary_name]
            description = find_result[0][description_name]
        if cls.__is_cache:
            cls.__cache[bug_id] = (summary, description)
        return summary, description

    def test_compare_stackinfo(self):
        from ir_log import IRLog
        from ir_config import IRConfig
        from ir_mongodb_helper import IRMongodbHelper
        from ir_gnome_st_tools import IRSTTools
        from ir_text import IRText
        from random import randint
        import pymongo

        IRLog.get_instance().start_log()
        IRConfig.get_instance().load('../data/test/bug_test.cfg')
        IRText.parse_info_level1('../data/test/stacktrace_test')
        
        con = IRMongodbHelper.get_instance().get_connection()
        db = con[IRConfig.get_instance().get('bug_db_name')]
        assert None != db
        col = db[IRConfig.get_instance().get('bug_text_collection_name')]
        assert None != col

        bugs = col.find()
        total = col.count()
        st1 = bugs[0]["stacktrace"]

        for i in range(total):
            st2 = bugs[i]["stacktrace"]
            result_weight = IRSTTools.compare_stackinfo(st1, st2, 'weight')
            result_max = IRSTTools.compare_stackinfo(st1, st2, 'max')
            IRLog.get_instance().println('Weight: %f, Max: %f' \
                    % (result_weight, result_max))

        IRLog.get_instance().stop_log()

    def test_filter(self):

        from ir_log import IRLog
        from ir_config import IRConfig
        from ir_mongodb_helper import IRMongodbHelper
        from ir_gnome_st_tools import IRSTTools
        from ir_text import IRText
        import pymongo

        IRLog.get_instance().start_log()
        IRConfig.get_instance().load('../data/test/bug_test.cfg')
        IRText.parse_info_level1('../data/test/info_level1_test')
        
        con = IRMongodbHelper.get_instance().get_connection()
        db = con[IRConfig.get_instance().get('bug_db_name')]
        assert None != db
        col = db[IRConfig.get_instance().get('bug_text_collection_name')]
        assert None != col
        # Maybe a bug here:
        # The test of filter (originally) depends on parse_info_level1
        # But parse_info_level1 seems to invoke filter...
        for bug in col.find():
            # TODO: it's not correct. no stacktrace in desc
            desc, stack = IRSTTools.filter(bug["desc"])      


        IRLog.get_instance().stop_log()

 def batch_generate_term_count(cls):
     """Generate term count for text in mongodb database,
         and store to database.
     """
     from ir_log import IRProgressBar
     from ir_text import IRText
     from ir_config import IRConfig
     from ir_mongodb_helper import IRCollection
     # config
     bug_id_name = IRConfig.get_instance().get('bug_id_name', 'bug_id')
     summary_name = IRConfig.get_instance().get('bug_summary_name', 'summ')
     description_name = IRConfig.get_instance().\
             get('bug_description_name', 'desc')
     
     termcount_collection = IRCollection(
         'bug_db_name', 'bug_termcount_collection_name', 'w')
     def iter_text(bug):
         summary_bow, description_bow = cls.calculate_term_count(
             bug[summary_name], bug[description_name])
         termcount_collection.insert({
             bug_id_name : bug[bug_id_name],
             summary_name : summary_bow,
             description_name : description_bow })
     IRProgressBar.execute_iteration_for_cursor(IRText.get_iterator({}),
                                                iter_text, "From Text to Term Count")
     termcount_collection.create_index([(bug_id_name, IRCollection.ASCENDING)])
     termcount_collection.close()

    def get_termcount_of_bug(cls, bug_id):
        """Get termcount of a bug

        Args:
            bug_id: int

        Returns:
            [dict, dict], [termcount of summary, termcount of description]
        """

        from ir_config import IRConfig
        from ir_mongodb_helper import IRCollection
        if cls.__is_cache:
            if bug_id in cls.__cache_term_count:
                return cls.__cache_term_count[bug_id]
        bug_id_name = IRConfig.get_instance().get('bug_id_name')
        summary_name = IRConfig.get_instance().get('bug_summary_name')
        description_name = IRConfig.get_instance().get('bug_description_name')
        termcount_collection = IRCollection(
            'bug_db_name', 'bug_termcount_collection_name', 'r')
        res = termcount_collection.find({bug_id_name : bug_id})
        summary = {}
        description = {}
        if res.count() > 0:
            summary = res[0][summary_name]
            description = res[0][description_name]
        if cls.__is_cache:
            cls.__cache_term_count[bug_id] = (summary, description)
        return summary, description

    def test_get_report_difference(self):
        from ir_log import IRLog
        from ir_config import IRConfig
        from ir_report import IRReport
        from ir_recommender import IRRecommender

        IRConfig.get_instance().load('../data/test/bug_test.cfg')

        new_report = IRReport('apple for summary', 'linux description')
        sim_report = IRReport('apple of ghost crashed', 'description linux wow')

        (diff_sum, diff_desc) = \
                IRRecommender.get_report_difference(new_report, sim_report)
        IRLog.get_instance().println('New summary: %s' \
                % (new_report.get_summary_text()))
        IRLog.get_instance().println('Sim summary: %s' \
                % (sim_report.get_summary_text()))
        IRLog.get_instance().println('New description: %s' \
                % (new_report.get_description_text()))
        IRLog.get_instance().println('Sim description: %s' \
                % (sim_report.get_description_text()))
        IRLog.get_instance().println('Diff of summary: %s' % (diff_sum))
        IRLog.get_instance().println('Diff of description: %s' % (diff_desc))
        assert diff_sum == {'ghost', 'crash'}
        assert diff_desc == {'wow'}

    def test_create_new_report_from_string(self):
        from nose.tools import eq_
        from ir_log import IRLog
        from ir_config import IRConfig
        from ir_report import IRReport
        from ir_term_count import IRTermCount

        IRLog.get_instance().start_log()
        IRConfig.get_instance().load('../data/test/bug_test.cfg')
        summary_text = 'Firefox crashed'
        description_text = 'When I was openning history folder, the f**king' \
                ' Firefox just crashed!\n'
        report = IRReport(summary_text, description_text)
        report.set_basic_info(12345, 'core')
        report.set_penalty_terms(IRTermCount.do_stemming(['ie', 'explore']))
        report.set_exclude_report_ids([100100])
        report.set_dummy_bug_id(12345)
        report.set_skip_terms(IRTermCount.do_stemming(['new','please']))
        # save to text
        text = report.to_string()
        IRLog.get_instance().println('Serialized report: %s' % (text))
        # load from text
        new_report = IRReport.from_string(text)

        assert new_report.get_summary_text() == report.get_summary_text()
        eq_(new_report.get_description_text().strip(), report.get_description_text().strip())
        assert new_report.get_create_ts() == report.get_create_ts()
        assert new_report.get_product() == report.get_product()
        assert new_report.get_dummy_bug_id() == report.get_dummy_bug_id()
        assert new_report.get_penalty_terms() == report.get_penalty_terms()
        assert new_report.get_exclude_report_ids() == report.get_exclude_report_ids()
        eq_(new_report.get_skip_terms(), report.get_skip_terms())
        IRLog.get_instance().stop_log()

    def test_parse_info_level1(self):
        #import sys
        #sys.path.append('../bin/')
        from ir_log import IRLog
        from ir_text import IRText
        from ir_config import IRConfig
        from ir_mongodb_helper import IRMongodbHelper

        IRLog.get_instance().start_log()
        IRConfig.get_instance().load('../data/test/bug_test.cfg')
        assert None != IRConfig.get_instance()
        IRText.parse_info_level1('../data/test/info_level1_test')
        IRLog.get_instance().stop_log()

        con = IRMongodbHelper.get_instance().get_connection()
        db = con[IRConfig.get_instance().get('bug_db_name')]
        assert None != db
        col = db[IRConfig.get_instance().get('bug_text_collection_name')]
        assert None != col
        # in the test data, we have 1000 in total.
        # within, 40 have no resolution, 154 are incomplete
        assert 833 == col.count()
        assert 'gnome is full of bugs ! (100000 currently)' == \
                col.find({'bug_id':100000})[0]["summ"]
        
        res = col.find({"summ":{'$regex':'(&gt)|(&lt)|(&quot)|(&apo)s|(&amp)'}})
        assert res.count() == 0

    def test_get_collection_status(self):
        
        from ir_config import IRConfig
        from ir_mongodb_helper import IRMongodbHelper
        dbhelper = IRMongodbHelper.get_instance()


        IRConfig.get_instance().load('../data/test/bug_test.cfg')
        collection = dbhelper.get_collection(
            'bug_db_name',
            'bug_mongodb_helper_collection_name',
                True)
        ts, success = dbhelper.get_collection_status(
            'bug_db_name',
            'bug_mongodb_helper_collection_name')
        assert success == False

        db_name = IRConfig.get_instance().get('bug_db_name')
        collection_name = IRConfig.get_instance(). \
                get('bug_mongodb_helper_collection_name')
        dbhelper.update_meta( db_name, collection_name, True)
        ts, success = dbhelper.get_collection_status(
            'bug_db_name',
            'bug_mongodb_helper_collection_name')
        assert success == True

    def calculate_tfidf_for_report_termcount(cls,
                                             summary_termcount,
                                             description_termcount):
        """Calculate TFIDF for single report.
        
        Args:
            summary_termcount: dict, {term -> termcount}
            description_termcount: dict, {term -> termcount}

        Returns:
            [dict, dict], [tfidf of summary, tfidf of description]
        """
        from ir_config import IRConfig
        from ir_mongodb_helper import IRCollection
        documentcount_collection = IRCollection(
            'bug_db_name', 'bug_documentcount_collection_name', 'r')
        total_document = cls.get_total_report_number()
        summary_tfidf = cls.calculate_tfidf(
            summary_termcount,
            IRConfig.get_instance().get('bug_summary_name'),
            total_document,
            documentcount_collection)
        description_tfidf = cls.calculate_tfidf(
            description_termcount,
            IRConfig.get_instance().get('bug_description_name'),
            total_document,
            documentcount_collection)
        return summary_tfidf, description_tfidf

    def get_summary_and_description_of_bug(cls, bug_id):
        """Get summary and description from mongodb.

        Args:
            bug_id: int

        Returns:
            [str, str], [summary, description]
        """
        from ir_config import IRConfig
        from ir_mongodb_helper import IRCollection
        if cls.__is_cache:
            if bug_id in cls.__cache_summary_description:
                return cls.__cache_summary_description[bug_id]
        bug_id_name = IRConfig.get_instance().get('bug_id_name')
        summary_name = IRConfig.get_instance().get('bug_summary_name')
        description_name = IRConfig.get_instance().get('bug_description_name')
        text_collection = IRCollection(
            'bug_db_name', 'bug_text_collection_name', 'r')
        res = text_collection.find({bug_id_name : bug_id})
        summary = ''
        description = ''
        if res.count() > 0:
            summary = res[0][summary_name]
            description = res[0][description_name]
        if cls.__is_cache:
            cls.__cache_summary_description[bug_id] = (summary, description)
        return summary, description

 def test_get_stacktrace_text_of_bug(self):
     from ir_log import IRLog
     from ir_config import IRConfig
     from ir_text import IRText
     IRConfig.get_instance().load('../data/test/bug_test.cfg')
     stacktrace_text = IRText.get_stacktrace_text_of_bug(104400)
     IRLog.get_instance().println('stacktrace_text: %s' % (stacktrace_text))