Python ir_log.IRLog类代码示例

 def test_get_stacktrace_text_of_bug(self):
     from ir_log import IRLog
     from ir_config import IRConfig
     from ir_text import IRText
     IRConfig.get_instance().load('../data/test/bug_test.cfg')
     stacktrace_text = IRText.get_stacktrace_text_of_bug(104400)
     IRLog.get_instance().println('stacktrace_text: %s' % (stacktrace_text))

    def show_dict_compare(cls, dicta, dictb, log_level = 1):
        """Compare the print two BoW.

        Args:
            dicta: dict, term -> count
            dictb: dict
            log_level: int
        """

        from ir_log import IRLog
        keys = set()
        if None != dicta:
            for key in dicta:
                keys.add(key)
        if None != dictb:
            for key in dictb:
                keys.add(key)
        # sort by common num
        common_num = []
        for key in keys:
            counta = 0
            countb = 0
            if None != dicta:
                if key in dicta:
                    counta = dicta[key]
            if None != dictb:
                if key in dictb:
                    countb = dictb[key]
            common_num.append((key, min(counta, countb), counta, countb))
        common_num.sort(cmp=lambda a,b:cmp(a[1],b[1]), reverse=True)
        # print it out
        for item in common_num:
            IRLog.get_instance().println('%16s\t%8d\t%8d' \
                        % (item[0], item[2], item[3]), log_level)

    def test_filter(self):

        from ir_log import IRLog
        from ir_config import IRConfig
        from ir_mongodb_helper import IRMongodbHelper
        from ir_gnome_st_tools import IRSTTools
        from ir_text import IRText
        import pymongo

        IRLog.get_instance().start_log()
        IRConfig.get_instance().load('../data/test/bug_test.cfg')
        IRText.parse_info_level1('../data/test/info_level1_test')
        
        con = IRMongodbHelper.get_instance().get_connection()
        db = con[IRConfig.get_instance().get('bug_db_name')]
        assert None != db
        col = db[IRConfig.get_instance().get('bug_text_collection_name')]
        assert None != col
        # Maybe a bug here:
        # The test of filter (originally) depends on parse_info_level1
        # But parse_info_level1 seems to invoke filter...
        for bug in col.find():
            # TODO: it's not correct. no stacktrace in desc
            desc, stack = IRSTTools.filter(bug["desc"])      


        IRLog.get_instance().stop_log()

    def __show_similarity_distribution(self, sorted_similarities):
        """Show the distribtuion of similarities.

        Args:
            sorted_similarities: [(bug_id, (score, ...))]
        """
        from ir_log import IRLog
        tot = sorted_similarities.__len__()
        # number of near top
        print sorted_similarities[0]
        max_score = sorted_similarities[0][1][0]
        min_score = sorted_similarities[-1][1][0]
        score_span = 0.1
        near_threshold = max_score - (max_score - min_score) * score_span
        near_one_number = 0
        for item in sorted_similarities:
            if item[1][0] > near_threshold:
                near_one_number += 1
            else:
                break
        IRLog.get_instance().println('%d in %d (%f) reports have score ' \
                'greater than %f (%f of the score span)' % \
                (near_one_number, tot, float(near_one_number)/tot,
                 near_threshold, score_span))
        # quantiles
        quantiles = [0.0, 0.2, 0.4, 0.6, 0.8, 1.0]
        for quan in quantiles:
            pos = int(quan * tot)
            if pos >= tot:
                pos = tot-1
            IRLog.get_instance().println('Top %d: %f' \
                    % (int(quan*100), sorted_similarities[pos][1][0]))

    def test_parse_info_level1(self):
        #import sys
        #sys.path.append('../bin/')
        from ir_log import IRLog
        from ir_text import IRText
        from ir_config import IRConfig
        from ir_mongodb_helper import IRMongodbHelper

        IRLog.get_instance().start_log()
        IRConfig.get_instance().load('../data/test/bug_test.cfg')
        assert None != IRConfig.get_instance()
        IRText.parse_info_level1('../data/test/info_level1_test')
        IRLog.get_instance().stop_log()

        con = IRMongodbHelper.get_instance().get_connection()
        db = con[IRConfig.get_instance().get('bug_db_name')]
        assert None != db
        col = db[IRConfig.get_instance().get('bug_text_collection_name')]
        assert None != col
        # in the test data, we have 1000 in total.
        # within, 40 have no resolution, 154 are incomplete
        assert 833 == col.count()
        assert 'gnome is full of bugs ! (100000 currently)' == \
                col.find({'bug_id':100000})[0]["summ"]
        
        res = col.find({"summ":{'$regex':'(&gt)|(&lt)|(&quot)|(&apo)s|(&amp)'}})
        assert res.count() == 0

 def run(self):
     from ir_log import IRLog
     session_state = STATE_ALIVE
     while session_state == STATE_ALIVE:
         try:
             msgpack = self.__msg_queue.get(True)
             # do something to msgpack
             conn = msgpack['connection']
             respack = msgpack['respack']
             respack[SESSION_ID] = msgpack[SESSION_ID]
             # set phase
             for key, value in msgpack.items():
                 if key in SET_COMMANDS:
                     self.__report = SET_COMMANDS[key](self.__report, value)
             # do phase
             signal = SIGNAL_CONTINUE
             for key, value in msgpack.items():
                 if key in CTL_COMMANDS:
                     signal = CTL_COMMANDS[key](self.__report, respack)
                     if signal == SIGNAL_BREAK:
                         session_state = STATE_EXPIRED
             self.__pack_report_info(respack)
             IRLog.get_instance().println('Send message: %s' % str(respack))
             conn.send(str(respack))
         except Queue.Empty:
             from ir_log import IRLog
             IRLog.get_instance().println('Session %d time out' % self.__id,
                                          2)
             break
     self.__dispatcher.remove_session(self.__id)

    def test_get_squared_length(self):
        from ir_log import IRLog
        from ir_config import IRConfig
        from ir_tfidf import IRTFIDF

        IRLog.get_instance().start_log()
        IRConfig.get_instance().load('../data/test/bug_test.cfg')
        summary = {'firefox':0.4, 'chrome':0.6}
        assert abs(IRTFIDF.get_squared_length(summary) - 0.52 ) < 0.00001

    def test_cache_all_data(self):
        from ir_log import IRLog
        from ir_config import IRConfig
        from ir_tfidf import IRTFIDF

        IRLog.get_instance().start_log()
        IRConfig.get_instance().load('../data/test/bug_test.cfg')
        IRTFIDF.cache_all_data()
        IRLog.get_instance().stop_log()

    def similarities_and_duplicates(self):
        """Calculate the similarities over all existing reports and return
        the similar reports and duplicate reports.

        Returns:
            [bug_id],[bug_id], [similar report ids],[duplicate report ids]
        """
        from ir_log import IRLog
        from ir_config import IRConfig
        from ir_duplicate_group import IRDuplicateGroup
        similar_threshold = IRConfig.get_instance().get_float(
            'bug_similar_threshold', 0.7)
        duplicate_num = IRConfig.get_instance().get_int(
            'bug_duplicate_number', 5)
        duplicate_threshold = IRConfig.get_instance().get_int(
            'bug_duplicate_threshold', 10)
        max_similar_number = IRConfig.get_instance().get_int(
            'bug_similar_max', 10000000)
        similar_threshold_percent = IRConfig.get_instance().get_float(
            'bug_similar_threshold_percent', 0.8)
        no_similar_threshold = IRConfig.get_instance().get_float(
            'bug_no_similar_threshold', 0.65)

        similarities = self.similarity_over_all().items()
        if similarities.__len__() == 0:
            return [], []
        similarities.sort(key=lambda  x:x[1][0], reverse = True)
        # report scoring
        IRLog.get_instance().println('Max score report: %s' % str(similarities[0]))
        if similarities.__len__() > 1:
            IRLog.get_instance().println('Second score report: %s' % str(similarities[1]))

        # find cutting edge of similar reports
        max_score = similarities[0][1][0]
        min_score = similarities[-1][1][0]
        IRLog.get_instance().println('max score:%f, min score: %f' %(max_score, min_score))
        IRLog.get_instance().println('no threshold:%f' % no_similar_threshold)
        if max_score < no_similar_threshold:
            return [], []

        similar_threshold_percent_cut = min_score + (max_score - min_score) *\
                                        similar_threshold_percent
        print 'cut:', similar_threshold_percent_cut

        cut_position = min(max_similar_number, self.__binary_search_less(similarities, lambda x:x[1][
            0], similar_threshold_percent_cut))
        IRLog.get_instance().println('Get %d similar reports.' % cut_position)
        # find number of duplicate groups in similar reports
        group_set = set()
        for report in similarities[:cut_position]:
            group_set.add(IRDuplicateGroup.get_group_of_bug(report[0]))
        if None in group_set:
            group_set.remove(None)
        duplicate_reports = []
        if group_set.__len__() <= duplicate_threshold:
            duplicate_reports = similarities[:min(cut_position, duplicate_num)]
        return similarities[:cut_position], duplicate_reports

    def test_get_summary_and_description_of_bug(self):

        from ir_log import IRLog
        from ir_config import IRConfig
        from ir_text import IRText
        IRConfig.get_instance().load('../data/test/bug_test.cfg')
        summary, description = IRText.get_summary_and_description_of_bug(100000)
        IRLog.get_instance().println('summary: %s' % (summary))
        IRLog.get_instance().println('description: %s' % (description))

    def print_similarity_score(cls, report_a, report_b):
        """Warning: report_a is primary! It is critial in asymatric algorithm"""

        from ir_log import IRLog

        total, summary, description, stacktrace = \
                report_a.similarity_with(report_b)
        IRLog.get_instance().println('[Similarity] %f '\
                '=[Summary]%f[Description]%f[Stacktrace]%f' \
                % (total, summary, description, stacktrace))

 def get_int(self, name, default_value = None):
     """Get the int value with the given name."""
     try:
         res = self.get(name, default_value)
         return int(res)
     except ValueError:
         from ir_log import IRLog
         IRLog.get_instance().println('Could not convert %d to int.' \
                 % (self.get(name)))
         return default_value

    def test_batch_generate_tfidf(self):
        #import sys
        #sys.path.append('../bin/')
        from ir_log import IRLog
        from ir_config import IRConfig
        from ir_tfidf import IRTFIDF

        IRLog.get_instance().start_log()
        IRConfig.get_instance().load('../data/test/bug_test.cfg')
        IRTFIDF.batch_generate_tfidf()
        IRLog.get_instance().stop_log()

def server_cache(msg, res):
    from ir_log import IRLog
    from ir_text import IRText
    from ir_tfidf import IRTFIDF
    from ir_document_count import IRDocumentCount
    IRLog.get_instance().println('Server is caching data')
    IRText.cache_all_data()
    IRTFIDF.cache_all_data()
    IRDocumentCount.cache_all_data()
    IRLog.get_instance().println('Server cached data')
    return SIGNAL_CONTINUE

    def similarity_over_all(self):
        """Calculate similarity between bug (summary, description) over
         all.

        Returns:
            dict, {bug_id -> [score, summary_score, description_score, stacktrace_score]}
        """

        from ir_log import IRLog
        from ir_config import IRConfig
        from ir_mongodb_helper import IRCollection
        from ir_text import IRText
        from ir_tfidf import IRTFIDF

        logger = IRLog.get_instance()
        search_time_span = 2 * 3600 * 24 * 365
        
        bug_id_name = IRConfig.get_instance().get('bug_id_name')
        
        create_ts_name = IRConfig.get_instance().get('bug_create_ts_name')
        product_name = IRConfig.get_instance().get('bug_product_name')

        basic_collection = IRCollection(
            'bug_db_name', 'bug_basic_collection_name', 'r')
        
        reports2scan = basic_collection.find({
            product_name : self.get_product(),
            create_ts_name : {'$gt' : self.get_create_ts() - search_time_span},
            bug_id_name : {'$nin' : self.__exclude_report_ids} })
        result = {}
        IRLog.get_instance().println('Comparing with %d reports.' \
                % (reports2scan.count()) )
        
        print self.__summary_text
        print self.__description_text

        for report in reports2scan:
            bug_id = report[bug_id_name]
            if bug_id == self.get_dummy_bug_id():
                continue
            # because we don't want to load stacktrace in case of self.__stacktrace 
            #    being none, we create and fill the info of report manually
            other_report = IRReport("", "")
            other_report.__summary_tfidf, other_report.__description_tfidf = \
                    IRTFIDF.get_tfidf_of_bug(bug_id)
            # if self.__stacktrace is empty, we don't need to do this
            if self.get_stacktrace() is not None and \
                    self.get_stacktrace().__len__() > 0:
                other_report.__stacktrace = IRText.get_stacktrace_of_bug(bug_id)
            if other_report.__stacktrace is None:
                other_report.__stacktrace = []
            result[bug_id] = self.similarity_with(other_report)

        return result