Python re2.compile函数代码示例

    def __init__(self,regular_expressions):
        """ Initialize the object 

            regular_expressions: an interative set of regular expressions to
                be applied for extracting mutations. These are in the 
                default python syntax (i.e., perl regular expressions), with 
                the single exception being that regular expressions which
                should be performed in a case sensitive manner should be 
                followed by the string '[CASE_SENSITIVE]', with no spaces 
                between it and the regular expression. 
                This can be a list, a file, or any other object which supports
                iteration. For an example, you should refer to the regex.txt
                file in the MutationFinder directory.

        """
        MutationExtractor.__init__(self)
        self._regular_expressions = []

        for regular_expression in regular_expressions:
            if regular_expression.endswith('[CASE_SENSITIVE]'):
                self._regular_expressions.append(\
                 compile(regular_expression[:regular_expression.rindex('[')]))
            else:
                self._regular_expressions.append(\
                 compile(regular_expression,IGNORECASE))

    def run(self):
        if self.results["target"]["category"] == "file":
            return False
 
        self.ie_paths_re = re.compile(r"^c:\\program files(?:\s\(x86\))?\\internet explorer\\iexplore.exe$",re.I)
        #run through re.escape()
        self.white_list_re = ["^C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Adobe\\\\Reader\\ \\d+\\.\\d+\\\\Reader\\\\AcroRd32\\.exe$",
                         "^C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Java\\\\jre\\d+\\\\bin\\\\j(?:avaw?|p2launcher)\\.exe$",
                         "^C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Microsoft SilverLight\\\\(?:\\d+\\.)+\\d\\\\agcp.exe$",
                         "^C\\:\\\\Windows\\\\System32\\\\ntvdm\\.exe$",
                         "^C\\:\\\\Windows\\\\system32\\\\rundll32\\.exe$",
                         "^C\\:\\\\Windows\\\\syswow64\\\\rundll32\\.exe$",
                         "^C\\:\\\\Windows\\\\system32\\\\drwtsn32\\.exe$",
                         "^C\\:\\\\Windows\\\\syswow64\\\\drwtsn32\\.exe$",
                         "^C\\:\\\\Windows\\\\system32\\\\dwwin\\.exe$"
                        ]
        #means we can be evaded but also means we can have relatively tight paths between 32-bit and 64-bit
        self.white_list_re_compiled = []
        for entry in self.white_list_re:
            self.white_list_re_compiled.append(re.compile(entry,re.I))
        self.white_list_re_compiled.append(self.ie_paths_re)

        # Sometimes if we get a service loaded we get out of order processes in tree need iterate over IE processes get the path of the initial monitored executable
        self.initialpath = None
        processes = self.results["behavior"]["processtree"]
        if len(processes):
            for p in processes:
                initialpath = p["module_path"].lower()
                if initialpath and self.ie_paths_re.match(initialpath) and p.has_key("children"):
                    self.martians = self.find_martians(p,self.white_list_re_compiled)
                    if len(self.martians) > 0:
                        for martian in self.martians:
                            self.data.append({"ie_martian": martian})
                        return True 
        return False

    def substitute(self, search, replace, flags, text, inputMessage, channel):
        # Apparently re.sub understands escape sequences in the replacement string;
        #  strip all but the backreferences
        replace = replace.replace('\\', '\\\\')
        replace = re.sub(r'\\([1-9][0-9]?([^0-9]|$))', r'\1', replace)

        if channel not in self.messages:
            self.messages[channel] = []
            self.unmodifiedMessages[channel] = []

        messages = self.unmodifiedMessages[channel] if 'o' in flags else self.messages[channel]

        if 'g' in flags:
            count = 0
        else:
            count = 1

        subFlags = 0
        if 'i' in flags:
            subFlags |= re.IGNORECASE
        if 'v' in flags:
            subFlags |= re.VERBOSE

        if 'c' in flags:
            newMessage = copy.copy(inputMessage)

            try:
                searchC = re2.compile(search, subFlags)
                new = searchC.sub(replace, text, count)
            except sre_constants.error as e:
                newMessage.messageString = "[Regex Error in Sed pattern: {}]".format(e.message)
                return newMessage

            if new != text:
                newMessage.messageString = new
                self.storeMessage(newMessage, False)
            else:
                newMessage.messageString = text
                self.storeMessage(newMessage, False)

            return newMessage

        for message in reversed(messages):
            try:
                searchC = re2.compile(search, subFlags)
                new = searchC.sub(replace, message.messageString, count)
            except sre_constants.error as e:
                newMessage = copy.copy(inputMessage)
                newMessage.messageString = "[Regex Error in Sed pattern: {}]".format(e.message)
                return newMessage

            new = new[:300]

            if searchC.search(message.messageString):
                newMessage = copy.copy(message)
                newMessage.messageString = new
                self.storeMessage(newMessage, False)
                return newMessage

        return None

    def _search(self, searchTerms, logPath, files, searchForNick, includeToday, reverse):
        if searchForNick:
            pattern = re2.compile(r"^\[[^]]+\]\s+<(.?{})>\s+.*".format(searchTerms), re.IGNORECASE)
        else:
            pattern = re2.compile(r".*<.*> .*({}).*".format(searchTerms), re.IGNORECASE)
        found = None

        if not includeToday:
            today = "{}.log".format(strftime("%Y-%m-%d"))
            if today in files:
                files.remove(today)

        if reverse:
            files.reverse()
        for filename in files:
            with open(os.path.join(logPath, filename), "r") as logfile:
                if reverse:
                    lines = reversed(logfile.readlines())
                else:
                    lines = logfile.readlines()
            if reverse and includeToday:
                lines = list(lines)[1:]
            for line in lines:
                if pattern.match(line.rstrip()):
                    found = line.rstrip()
                    break
            if found:
                return "[{}] {}".format(filename[:10], found)
        return "Nothing that matches your search terms has been found in the log."

    def test_re_match(self):
        self.assertEqual(re.match('a', 'a').groups(), ())
        self.assertEqual(re.match('(a)', 'a').groups(), ('a',))
        self.assertEqual(re.match(r'(a)', 'a').group(0), 'a')
        self.assertEqual(re.match(r'(a)', 'a').group(1), 'a')
        self.assertEqual(re.match(r'(a)', 'a').group(1, 1), ('a', 'a'))

        pat = re.compile('((a)|(b))(c)?')
        self.assertEqual(pat.match('a').groups(), ('a', 'a', None, None))
        self.assertEqual(pat.match('b').groups(), ('b', None, 'b', None))
        self.assertEqual(pat.match('ac').groups(), ('a', 'a', None, 'c'))
        self.assertEqual(pat.match('bc').groups(), ('b', None, 'b', 'c'))
        self.assertEqual(pat.match('bc').groups(""), ('b', "", 'b', 'c'))

        # A single group
        m = re.match('(a)', 'a')
        self.assertEqual(m.group(0), 'a')
        self.assertEqual(m.group(0), 'a')
        self.assertEqual(m.group(1), 'a')
        self.assertEqual(m.group(1, 1), ('a', 'a'))

        pat = re.compile('(?:(?P<a1>a)|(?P<b2>b))(?P<c3>c)?')
        self.assertEqual(pat.match('a').group(1, 2, 3), ('a', None, None))
        self.assertEqual(pat.match('b').group('a1', 'b2', 'c3'),
                         (None, 'b', None))
        self.assertEqual(pat.match('ac').group(1, 'b2', 3), ('a', None, 'c'))

 def test_bug_926075(self):
     try:
         unicode
     except NameError:
         return # no problem if we have no unicode
     self.assert_(re.compile('bug_926075') is not
                  re.compile(eval("u'bug_926075'")))

 def test_empty_array(self):
     # SF buf 1647541
     import array
     for typecode in 'cbBuhHiIlLfd':
         a = array.array(typecode)
         self.assertEqual(re.compile("bla").match(a), None)
         self.assertEqual(re.compile("").match(a).groups(), ())

    def run(self):
        self.ie_paths_re = re.compile(r"^c:\\program files(?:\s\(x86\))?\\internet explorer\\iexplore.exe$",re.I)
        #run through re.escape()
        self.white_list_re = ["^C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Adobe\\\\Reader\\ \\d+\\.\\d+\\\\Reader\\\\AcroRd32\\.exe$",
                         "^C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Java\\\\jre\\d+\\\\bin\\\\j(?:avaw?|p2launcher)\\.exe$",
                         "^C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Microsoft SilverLight\\\\(?:\\d+\\.)+\\d\\\\agcp.exe$",
                         "^C\\:\\\\Windows\\\\System32\\\\ntvdm.exe$",
                        ]
        #means we can be evaded but also means we can have relatively tight paths between 32-bit and 64-bit
        self.white_list_re_compiled = []
        for entry in self.white_list_re:
            self.white_list_re_compiled.append(re.compile(entry,re.I))
        self.white_list_re_compiled.append(self.ie_paths_re)

        # get the path of the initial monitored executable
        self.initialpath = None
        processes = self.results["behavior"]["processtree"]
        if len(processes):
            self.initialpath = processes[0]["module_path"].lower()
        if self.initialpath and self.ie_paths_re.match(self.initialpath) and processes[0].has_key("children"):
           self.martians = self.find_martians(processes,self.white_list_re_compiled)
           if len(self.martians) > 0:
               for martian in self.martians:
                   self.data.append({"ie_martian": martian})
               return True 
        return False

    def _search(self, searchTerms, logPath, files, searchForNick, includeToday, reverse):
        if searchForNick:
            pattern = re2.compile(fr"^\[[^]]+\]\s+<(.?{searchTerms})>\s+.*", re2.IGNORECASE)
        else:
            pattern = re2.compile(fr'.*<.*> .*({searchTerms}).*', re2.IGNORECASE)
        found = None

        if not includeToday:
            today = f"{strftime('%Y-%m-%d')}.log"
            if today in files:
                files.remove(today)

        if reverse:
            files.reverse()
        for filename in files:
            with open(os.path.join(logPath, filename), 'r', errors='ignore') as logfile:
                if reverse:
                    lines = reversed(logfile.readlines())
                else:
                    lines = logfile.readlines()
            if reverse and includeToday:
                lines = list(lines)[1:]
            for line in lines:
                if pattern.match(line.rstrip()):
                    found = line.rstrip()
                    break
            if found:
                return f'[{filename[:10]}] {found}'
        return 'Nothing that matches your search terms has been found in the log.'

 def __init__(self, *args, **kwargs):
     Signature.__init__(self, *args, **kwargs)
     # Named group to extract the URL of the cloned website.
     self.rex = {
         "saved from url": re.compile(r"\<!--\ssaved\sfrom\surl=\(\d+\)(?P<url>[^\s]+)", re.I),
         "mirrored from": re.compile(r"<!--\smirrored\sfrom\s(?P<url>[^\s]+)\sby\sHTTrack", re.I),
     }
     self.hits = set()

def test_regex(regex_array):
    """Ensures the regex strings are validated for proper syntax.

    """
    for regex_entry in regex_array:
        try:
            re.compile(regex_entry, re.MULTILINE | re.UNICODE)
        except re.error:
            logging.error('Invalid Regex Found: %s', regex_entry)
            sys.exit(1)

def _compile_regex(regex_string, flags=0):
    try:
        if re2:
            # default max_mem is 8<<20 = 8*1000*1000
            return re.compile(regex_string, max_mem=60*1000*1000, flags=flags)
        else:
            return re.compile(regex_string, flags=flags)
    except:
        logging.exception("Error compiling with flags %s for string: %s", flags, regex_string)
        raise

 def _prepare_pattern(self, pattern):
     """
     Strip out key:value pairs from the pattern and compile the regular expression.
     """
     regex, _, rest = pattern.partition('\\;')
     try:
         return re.compile(regex, re.I)
     except re.error as e:
         warnings.warn("Caught '{error}' compiling regex: {regex}".format(error=e, regex=regex))
         return re.compile(r'(?!x)x') # regex that never matches: http://stackoverflow.com/a/1845097/413622

    def test_dollar_matches_twice(self):
        "$ matches the end of string, and just before the terminating \n"
        pattern = re.compile('$')
        self.assertEqual(pattern.sub('#', 'a\nb\n'), 'a\nb#\n#')
        self.assertEqual(pattern.sub('#', 'a\nb\nc'), 'a\nb\nc#')
        self.assertEqual(pattern.sub('#', '\n'), '#\n#')

        pattern = re.compile('$', re.MULTILINE)
        self.assertEqual(pattern.sub('#', 'a\nb\n' ), 'a#\nb#\n#' )
        self.assertEqual(pattern.sub('#', 'a\nb\nc'), 'a#\nb#\nc#')
        self.assertEqual(pattern.sub('#', '\n'), '#\n#')

    def run(self):
        if self.results["target"]["category"] == "url":
            return False

        office_pkgs = ["ppt","doc","xls","eml"]
        if not any(e in self.results["info"]["package"] for e in office_pkgs):
            return False

        self.office_paths_re = re.compile(r"^[A-Z]\:\\Program Files(?:\s\(x86\))?\\Microsoft Office\\(?:Office1[1-5]\\)?(?:WINWORD|OUTLOOK|POWERPNT|EXCEL|WORDVIEW)\.EXE$",re.I)
        #run through re.escape()
        #############################################
        #YOU MAY HAVE TO CUSTOMIZE THIS FOR YOUR ENV#
        #############################################
        self.white_list_re = ["C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Adobe\\\\Reader\\ \\d+\\.\\d+\\\\Reader\\\\AcroRd32\\.exe$",
                         "C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Java\\\\jre\\d+\\\\bin\\\\j(?:avaw?|p2launcher)\\.exe$",
                         "C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Microsoft SilverLight\\\\(?:\\d+\\.)+\\d\\\\agcp\\.exe$",
                         "C\\:\\\\Windows\\\\System32\\\\ntvdm\\.exe$",
                         "C\\:\\\\Windows\\\\System32\\\\svchost\\.exe$",
                         "C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\internet explorer\\\\iexplore\.exe$",
                         # remove this one at some point
                         "C\\:\\\\Windows\\\\System32\\\\rundll32\\.exe$",
                         "C\\:\\\\Windows\\\\System32\\\\drwtsn32\\.exe$",
                         "C\\:\\\\Windows\\\\splwow64\\.exe$",
                         "C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Common Files\\\\Microsoft Shared\\\\office1[1-6]\\\\off(?:lb|diag)\\.exe$",
                         "C\\:\\\\Program Files(?:\s\\(x86\\))?\\\\Common Files\\\\Microsoft Shared\\\\dw\\\\dw(?:20)?\\.exe$",
                         "C\\:\\\\Windows\\\\system32\\\\dwwin\\.exe$",
                         "C\\:\\\\Windows\\\\system32\\\\WerFault\\.exe$",
                         "C\\:\\\\Windows\\\\syswow64\\\\WerFault\\.exe$"
                        ]
        #means we can be evaded but also means we can have relatively tight paths between 32-bit and 64-bit
        self.white_list_re_compiled = []
        for entry in self.white_list_re:
            try:
                self.white_list_re_compiled.append(re.compile(entry,re.I))
            except Exception as e:
                print "failed to compile expression %s error:%s" % (entry,e)
        self.white_list_re_compiled.append(self.office_paths_re)

        # Sometimes if we get a service loaded we get out of order processes in tree need iterate over IE processes get the path of the initial monitored executable
        self.initialpath = None
        processes = self.results["behavior"]["processtree"]
        if len(processes):
            for p in processes:
                initialpath = p["module_path"].lower()
                if initialpath and self.office_paths_re.match(initialpath) and p.has_key("children"):
                    self.martians = self.find_martians(p,self.white_list_re_compiled)
                    if len(self.martians) > 0:
                        for martian in self.martians:
                            self.data.append({"office_martian": martian})
                        return True
        return False