Python pygments.lex函数代码示例

def findMain(code):
    """
    Look for the existence of if __name__ == '__main__'
    Documentation: https://docs.python.org/2/tutorial/modules.html in 6.1.1
    """
    found = False
    pos = 0
    lexer = PythonLexer()

    tokens_1 = pygments.lex(code, lexer)
    tokens_2 = pygments.lex(code, lexer)
    
    sequence_1 = [(Token.Keyword, '^if$'),
                (Token.Name, '^__name__$'),
                (Token.Operator, '^==$'),
                (Token.Literal.String.Double, '^__main__$'),
                (Token.Punctuation, '^:$')]

    sequence_2 = [(Token.Keyword, '^if$'),
                (Token.Name, '^__name__$'),
                (Token.Operator, '^==$'),
                (Token.Literal.String.Single, '^__main__$'),
                (Token.Punctuation, '^:$')]

    mainIdiom = PythonIdiom('ifNameMain')

    lineNum = _findSeqInTokens(sequence_1, tokens_1)
    if lineNum < 0:
        lineNum = _findSeqInTokens(sequence_2, tokens_2)
    if lineNum > 0:
        mainIdiom.addNew(lineNum)
    log("If name main found in lines: " + str(mainIdiom.getLines()))
    return mainIdiom

def test_bare_class_handler():
    from pygments.formatters import HtmlFormatter
    from pygments.lexers import PythonLexer
    try:
        lex('test\n', PythonLexer)
    except TypeError as e:
        assert 'lex() argument must be a lexer instance' in str(e)
    else:
        assert False, 'nothing raised'
    try:
        format([], HtmlFormatter)
    except TypeError as e:
        assert 'format() argument must be a formatter instance' in str(e)
    else:
        assert False, 'nothing raised'

def findMagicMethods(code):
    """
    Search magic methods in the code and returns a list of how many have
    been found, what kind of dificult it has and wich where
    Documentation: http://www.rafekettler.com/magicmethods.html
                   Python Pocket Reference page 88
    """
    lexer = PythonLexer()
    tokens = pygments.lex(code, lexer)
    lineNumber = 1
    methodsFound = []
    methodsIdiom1 = PythonIdiom('idiomMethods1')
    methodsIdiom2 = PythonIdiom('idiomMethods2')
    methodsIdiom3 = PythonIdiom('idiomMethods3')

    for ttype, word in tokens:
        lineNumber += _getNewLines((ttype, word))
        if ttype is Token.Name.Function:
            if word in magicMethods_1:
                methodsIdiom1.addNew(lineNumber, otherInfo={'method': word})
                methodsFound.append(word)
            elif word in magicMethods_2:
                methodsIdiom2.addNew(lineNumber, otherInfo={'method': word})
                methodsFound.append(word)
            elif word in magicMethods_3:
                methodsIdiom3.addNew(lineNumber, otherInfo={'method': word})
                methodsFound.append(word)

    log("MagicMethods: %s" % str(methodsFound))
    return [methodsIdiom1, methodsIdiom2, methodsIdiom3]

def basicStructure(code):
    sequence = []
    lexer = PythonLexer()
    lexer.add_filter('tokenmerge')
    tokens = pygments.lex(code, lexer)
    for token in tokens:
        print token

    def __init__(self, pdf, code, lexer):
        self.pdf = pdf
        fname, fstyle, fsize = self.pdf.theme["code-font"]

        self.pdf.set_font(fname, fstyle, fsize)
        style = pygments.styles.get_style_by_name("emacs")
        style = dict(style)
        for token, text in pygments.lex(code["code"], lexer):
            token_style = style[token]

            if token_style["color"]:
                r, g, b = map(ord, token_style["color"].decode("hex"))
            else:
                r, g, b = (0, 0, 0)
            self.pdf.set_text_color(r, g, b)

            if token_style["bold"] and token_style["italic"]:
                self.pdf.set_font(fname, "BI", fsize)
            elif token_style["bold"]:
                self.pdf.set_font(fname, "B", fsize)
            elif token_style["italic"]:
                self.pdf.set_font(fname, "I", fsize)
            else:
                self.pdf.set_font(fname, "", fsize)

            height = pdf.theme["code-height"]
            self.pdf.write(height, text)

def findDocstring(code):
    """Find the use of documentation in the functions, classes or script
    Documentation: https://www.python.org/dev/peps/pep-0257/
    """
    lexer = PythonLexer()
    lexer.add_filter('tokenmerge')

    classDefToken = (Token.Keyword, '^class$')
    functDefToken = (Token.Keyword, '^def$')
    tokens = pygments.lex(code, lexer)

    docIdiom = PythonIdiom('docstring')
    docstringFound = defaultdict(int)
    typeDoc = 'module'
    lineNumber = 1


    for ttype, word in tokens:
        if _sameToken((ttype, word), classDefToken):
            typeDoc = 'class'
        elif _sameToken((ttype, word), functDefToken):
            typeDoc = 'function'
        elif ttype == Token.Literal.String.Doc:
            docstringFound[typeDoc] += 1
            docIdiom.addNew(lineNumber)
        lineNumber += _getNewLines((ttype, word))

    for typeDoc in docstringFound:
        log("type %s: %d found" % (typeDoc, docstringFound[typeDoc]))
    log('DocString found in lines: ' + str(docIdiom.getLines()))
    return docIdiom

def check(*expected):
    text = ''.join(i[1] for i in expected)
    md_lexer = MarkdownLexer()
    md_lexer.add_filter('raiseonerror')
    md_lexer.add_filter('tokenmerge')
    result = list(pygments.lex(text, md_lexer))
    assert result == list(expected)

 def SyntexHighlight(self, event=None):
     from tkinter.font import Font
     for tag in self.tag_names():
         self.tag_delete(tag)
     self.mark_set("range_start", "1.0")
     data = self._get_value()
     self.tag_configure("Token.Comment", foreground="#F00")
     bolder = Font(family=self.app.cnf['font'][0])
     bolder.config(size=self.app.cnf['font'][1]-2)
     bolder.config(weight="bold")
     for token, content in lex(data, PythonLexer()):
         self.mark_set("range_end", "range_start + %dc" % len(content))
         self.tag_add(str(token), "range_start", "range_end")
         self.mark_set("range_start", "range_end")
     self.tag_config("Token.Comment.Single", foreground="#F00")
     self.tag_config("Token.Literal.String.Doc", foreground="#F00")
     for tag in self.tag_names():
         if 'Token.Keyword' == tag:
             self.tag_config(tag, foreground="#008", font=bolder)
         elif 'Token.Keyword.Namespace' == tag:
             self.tag_config(tag, foreground="#00F", font=bolder)
         elif 'Token.Name.Class' in tag:
             self.tag_config(tag, foreground="#F30", background='#AFA')
         elif 'Token.Name.Function' in tag:
             self.tag_config(tag, foreground="#A3A", background='#FFA')
         elif 'Token.Literal' in tag:
             self.tag_config(tag, foreground="#6A0")
         elif 'Token.Operator' in tag:
             self.tag_config(tag, foreground="#A3A")
     print(self.tag_names())

    def _lexContents(self):
        # We add a space in front because otherwise the lexer will discard
        # everything up to the first token, meaning that we lose the potentially
        # empty first lines and mess up the matching. With the space, we force
        # the lexer to process the initial \n. and we just skip the space token
        tokens = list(pygments.lex(" "+self._document.documentText(), pygments.lexers.PythonLexer()))
        self._document.beginTransaction()
        current_line_num = 1
        meta = []

        # Skip the space token
        for token in tokens[1:]:
            ttype, string = token

            meta.extend([ttype]*len(string))

            if string.endswith('\n'):
                self._document.deleteCharMeta( (current_line_num,1),
                                                self._document.lineLength(current_line_num),
                                                CharMeta.LexerToken)
                self._document.updateCharMeta((current_line_num,1), {CharMeta.LexerToken: meta})
                current_line_num += 1
                meta = []

        self._document.endTransaction()

    def filename(self, value):
        "Set the file being displayed by the view"
        if self._filename != value:
            self.code.delete('1.0', END)
            with open(value) as code:
                all_content = code.read()
                if self.lexer:
                    lexer = self.lexer
                else:
                    lexer = guess_lexer_for_filename(value, all_content, stripnl=False)
                for token, content in lex(all_content, lexer):
                    self.code.insert(END, content, str(token))

            # Now update the text for the linenumbers
            end_index = self.code.index(END)
            line_count = int(end_index.split('.')[0])
            lineNumbers = '\n'.join('%5d' % i for i in range(1, line_count))
            self.lines.config(state=NORMAL)
            self.lines.delete('1.0', END)
            self.lines.insert('1.0', lineNumbers)
            self.lines.config(state=DISABLED)

            # Store the new filename, and clear any current line
            self._filename = value
            self._line = None

def checkNotRange(code):
    """
    Check if there is: for xx in [0,1,2] instead of for xxx in (x)range
    Documentation: https://youtu.be/OSGv2VnC0go?t=3m4s
    """
    sequence = [(Token.Keyword, '^for$'),
                (Token.Name, '^\w+$'),
                (Token.Operator.Word, '^in$'),
                (Token.Punctuation, '^\[$'),
                (Token.Literal.Number.Integer, '^\d$')]

    lexer = PythonLexer()
    lexer.add_filter('tokenmerge')
    tokens = pygments.lex(code, lexer)
    notRangeIdiom = PythonIdiom('notRange')

    lineNumber = 1
    while True:
        lineAux = _findSeqInTokens(sequence, tokens)
        if lineAux < 0:
            break
        lineNumber += lineAux -1
        notRangeIdiom.addNew(lineNumber)
    log("badForIn found in lines {0}".format(notRangeIdiom.getLines()))
    return notRangeIdiom

    def __init__(self, disassembly, lexer=lexer, msg=None):

        self.lines = []
        if isinstance(disassembly, list):
            self.lines = disassembly
        elif disassembly:
            line = []
            if msg:
                current_function = msg.rsplit(None, 1)[-1][:-1]
            else:
                current_function = None
            with currentfunctiontfilter.current_function(current_function):
                for ttype, value in pygments.lex(disassembly, lexer):
                    if '\n' in value:
                        self.lines.append(DisassemblyLine(line))
                        line = []
                    else:
                        line.append((ttype, value))

        self.linenos = {}
        for i, line in enumerate(self.lines):
            self.linenos[line.address] = line, i

        self.lexer = lexer
        self.msg = msg

def highlightMultiSource(codeLexerTuples, multiSourceFormatter, outfile=None):
    """
    main function to create formatted output based on tuples of code and
    related metadata (lexing information and title to display)
    """
    if not isinstance(codeLexerTuples, tuple):
        raise TypeError(
            "first highlight() argument must be a tupel of \
codeLexerTuple"
        )
    if not isinstance(multiSourceFormatter, Formatter):
        raise TypeError(
            "second highlight() argument must be a \
MultiSourceFormatter"
        )

    tokensList = []
    for codeLexerTuple in codeLexerTuples:
        tokensList.append(lex(codeLexerTuple.code, codeLexerTuple.lexer))
        multiSourceFormatter.titles.append(codeLexerTuple.title)
    if not outfile:
        # print formatter, 'using', formatter.encoding
        realoutfile = multiSourceFormatter.encoding and BytesIO() or StringIO()
        multiSourceFormatter.format(tokensList, realoutfile)
        return realoutfile.getvalue()
    else:
        multiSourceFormatter.format(tokensList, outfile)

def style_ansi(raw_code, lang=None):
    """ actual code hilite """
    lexer = 0
    if lang:
        try:
            lexer = get_lexer_by_name(lang)
        except ValueError:
            print col(R, 'Lexer for %s not found' % lang)
    lexer = None
    if not lexer:
        try:
            if guess_lexer:
                lexer = pyg_guess_lexer(raw_code)
        except:
            pass
    if not lexer:
        lexer = get_lexer_by_name(def_lexer)
    tokens = lex(raw_code, lexer)
    cod = []
    for t, v in tokens:
        if not v:
            continue
        _col = code_hl_tokens.get(t)
        if _col:
            cod.append(col(v, _col))
        else:
            cod.append(v)
    return ''.join(cod)

def main():
    arguments = docopt( 
        __doc__.format( program=docstring_format_dict ), 
        version=        '{docstring_format_dict["human_format"]} 2.0', 
        options_first=  True
        )

    lexer = BibtexLexer()
    lexer.add_filter( RaiseOnErrorTokenFilter() )
    #lexer.add_filter( TokenMergeFilter() )
    lexer.add_filter( KeywordCaseFilter(case='lower') )
    
    for f in arguments['<file>']:
        # get bibtex source
        code = None
        with open(f, 'r') as f:
            code = ''.join( f.readlines() )

        # NOW LEX SEE CODE!
        for idx, item in enumerate(pygments.lex(code, lexer)):
            tokentype, tokenvalue = item[0], item[1]
            
            # if tokentype in frozenset([Token.Text.Whitespace, Token.Punctuation]):
            #     continue
            print(  "{0:>5}\t{1[0]!s:<25}\t{1[1]!r}".format(idx, item),
                    file=sys.stdout )