Python lexers.PythonLexer类代码示例

def checkBadLoopCollect(code):
    """
    Look for bad loop like 'for i in range(len(list))'
    Documentation: https://youtu.be/OSGv2VnC0go?t=4m47s
    """
    sequence = [(Token.Keyword, '^for$'),
                (Token.Name, '^\w+$'),
                (Token.Operator.Word, '^in$'),
                (Token.Name.Builtin, '^range$|^xrange$'),
                (Token.Punctuation, '^\($'),
                (Token.Name.Builtin, '^len$'),
                (Token.Punctuation, '^\($'),
                (Token.Name, '^\w+$')]
    lexer = PythonLexer()
    lexer.add_filter('tokenmerge')
    tokens = pygments.lex(code, lexer)
    badLoopCollectIdiom = PythonIdiom('badLoop')

    lineNumber = 1
    while True:
        lineAux = _findSeqInTokens(sequence, tokens)
        if lineAux < 0:
            break
        lineNumber += lineAux -1
        badLoopCollectIdiom.addNew(lineNumber)
    log("badLoopCollect found in lines {0}".format(badLoopCollectIdiom.getLines()))

    return badLoopCollectIdiom

def findBadUseImport(code):
    """
    Find when use from foo import *
    Documentation: http://python.net/~goodger/projects/pycon/2007/idiomatic/handout.html#importing
                   https://docs.python.org/2/howto/doanddont.html#from-module-import
    """
    sequence = [(Token.Keyword.Namespace, '^from$'),
                (Token.Name.Namespace, '.*'),
                (Token.Keyword.Namespace, '^import$'),
                (Token.Operator, '\*')]
    lexer = PythonLexer()
    lexer.add_filter('tokenmerge')
    tokens = pygments.lex(code, lexer)
    badUseImport = PythonIdiom('badImport')

    lineNumber = 1
    while True:
        lineAux = _findSeqInTokens(sequence, tokens)
        if lineAux < 0:
            break
        lineNumber += lineAux -1
        badUseImport.addNew(lineNumber)
    log("badUseImport found in lines {0}".format(badUseImport.getLines()))

    return badUseImport

def findDocstring(code):
    """Find the use of documentation in the functions, classes or script
    Documentation: https://www.python.org/dev/peps/pep-0257/
    """
    lexer = PythonLexer()
    lexer.add_filter('tokenmerge')

    classDefToken = (Token.Keyword, '^class$')
    functDefToken = (Token.Keyword, '^def$')
    tokens = pygments.lex(code, lexer)

    docIdiom = PythonIdiom('docstring')
    docstringFound = defaultdict(int)
    typeDoc = 'module'
    lineNumber = 1


    for ttype, word in tokens:
        if _sameToken((ttype, word), classDefToken):
            typeDoc = 'class'
        elif _sameToken((ttype, word), functDefToken):
            typeDoc = 'function'
        elif ttype == Token.Literal.String.Doc:
            docstringFound[typeDoc] += 1
            docIdiom.addNew(lineNumber)
        lineNumber += _getNewLines((ttype, word))

    for typeDoc in docstringFound:
        log("type %s: %d found" % (typeDoc, docstringFound[typeDoc]))
    log('DocString found in lines: ' + str(docIdiom.getLines()))
    return docIdiom

def checkNotRange(code):
    """
    Check if there is: for xx in [0,1,2] instead of for xxx in (x)range
    Documentation: https://youtu.be/OSGv2VnC0go?t=3m4s
    """
    sequence = [(Token.Keyword, '^for$'),
                (Token.Name, '^\w+$'),
                (Token.Operator.Word, '^in$'),
                (Token.Punctuation, '^\[$'),
                (Token.Literal.Number.Integer, '^\d$')]

    lexer = PythonLexer()
    lexer.add_filter('tokenmerge')
    tokens = pygments.lex(code, lexer)
    notRangeIdiom = PythonIdiom('notRange')

    lineNumber = 1
    while True:
        lineAux = _findSeqInTokens(sequence, tokens)
        if lineAux < 0:
            break
        lineNumber += lineAux -1
        notRangeIdiom.addNew(lineNumber)
    log("badForIn found in lines {0}".format(notRangeIdiom.getLines()))
    return notRangeIdiom

def python_line_tokens(code_lines, blank_lines=False):
    from pygments.lexers import PythonLexer
    lexer = PythonLexer()
    code_str = "".join(code_lines)
    all_tokens = list(lexer.get_tokens(code_str, unfiltered=True))
    line_tokens = []
    current_line = []

    for t in all_tokens:
        if t[1] == u"\n":
            line_tokens.append(current_line)
            current_line = []
        else:
            current_line.append(t)

    rows = []
    for i, tokens in enumerate(line_tokens):
        # Check for blank line
        line_str = code_lines[i].rstrip()
        if (not blank_lines) and len(line_str.strip()) == 0:
            continue

        for t in tokens:
            kind, value = str(t[0]), t[1]
            yield line_str, i, kind, value, t

def basicStructure(code):
    sequence = []
    lexer = PythonLexer()
    lexer.add_filter('tokenmerge')
    tokens = pygments.lex(code, lexer)
    for token in tokens:
        print token

def python_prettify(code, style):
    lexer = PythonLexer()
    lexer.add_filter(VisibleWhitespaceFilter(spaces="&nbsp"))
    pretty_code = highlight(
        code, lexer, HtmlFormatter(
            linenos=style, linenostart=0))
    # print(pretty_code)
    return format_html('{}', mark_safe(pretty_code))

def redent(s):
    """
    Shamelessly stolen from infobob(#python bot) code
    https://code.launchpad.net/~pound-python/infobat/infobob
    """
    lexer = PythonLexer()
    lexer.add_filter(_RedentFilter())
    return highlight(s, lexer, NullFormatter())

    def _analyse_source_code(self, source_code):

        lexer = PythonLexer()
        token_source = lexer.get_tokens(source_code)
        for token_type, value in token_source:
            if len(value) > 3 and value.startswith('gl') and ord('A') <= ord(value[2]) <= ord('Z'):
                self.gl_functions.add(value)
            elif len(value) > 3 and value.startswith('GL_'):
                self.gl_constants.add(value)

def python_token_metrics(code_lines, indent_size=4):
    from pygments.lexers import PythonLexer
    indent_regex = re.compile(r"^\s*")

    lexer = PythonLexer()
    code_str = "".join(code_lines)
    all_tokens = list(lexer.get_tokens(code_str, unfiltered=True))
    line_tokens = []
    current_line = []

    for t in all_tokens:
        if t[1] == u"\n":
            line_tokens.append(current_line)
            current_line = []
        else:
            current_line.append(t)

    rows = []
    for i, tokens in enumerate(line_tokens):
        line_number = i + 1

        # Check for blank line
        line_str = code_lines[i].rstrip()
        if len(line_str.strip()) == 0:
            rows.append([line_number, 0, 0, 0, 0, 0, 0])
            continue

        assert len(tokens) > 0, "No tokens for line"

        num_keywords = 0
        num_identifiers = 0
        num_operators = 0
        line_length = len(line_str)
        line_indent = len(indent_regex.findall(line_str)[0]) / indent_size

        # Indentation is not considered
        line_str_noindent = line_str.lstrip()
        line_length_noindent = len(line_str_noindent)
        whitespace_prop = line_str_noindent.count(" ") / float(line_length_noindent)

        for t in tokens:
            kind, value = str(t[0]), t[1]
            if kind.startswith(u"Token.Keyword"):
                num_keywords += 1
            elif kind.startswith(u"Token.Name"):
                num_identifiers += 1
            elif kind.startswith(u"Token.Operator"):
                num_operators += 1

        rows.append([line_number, line_length_noindent, num_keywords,
            num_identifiers, num_operators, whitespace_prop,
            line_indent])

    columns = ["line", "line_length", "keywords",
               "identifiers", "operators", "whitespace_prop",
               "line_indent"]
    return pandas.DataFrame(rows, columns=columns)

    def testWorksAsExpected(self):
        code = '''
        """ Increment number of decision points in function."""
        #if tok and tok.text in McCabeKeywords:
        if (tok[0][0] == b'Keyword') and tok[1] in McCabeKeywords:
            self.metrics['mccabe'] += 1
        '''
        result = [(Token.Text, u'        '), (Token.Literal.String.Doc, u'""" Increment number of decision points in function."""'), (Token.Text, u'\n'), (Token.Text, u'        '), (Token.Comment, u'#if tok and tok.text in McCabeKeywords:'), (Token.Text, u'\n'), (Token.Text, u'        '), (Token.Keyword, u'if'), (Token.Text, u' '), (Token.Punctuation, u'('), (Token.Name, u'tok'), (Token.Punctuation, u'['), (Token.Literal.Number.Integer, u'0'), (Token.Punctuation, u']'), (Token.Punctuation, u'['), (Token.Literal.Number.Integer, u'0'), (Token.Punctuation, u']'), (Token.Text, u' '), (Token.Operator, u'=='), (Token.Text, u' '), (Token.Name, u'b'), (Token.Literal.String, u"'"), (Token.Literal.String, u'Keyword'), (Token.Literal.String, u"'"), (Token.Punctuation, u')'), (Token.Text, u' '), (Token.Operator.Word, u'and'), (Token.Text, u' '), (Token.Name, u'tok'), (Token.Punctuation, u'['), (Token.Literal.Number.Integer, u'1'), (Token.Punctuation, u']'), (Token.Text, u' '), (Token.Operator.Word, u'in'), (Token.Text, u' '), (Token.Name, u'McCabeKeywords'), (Token.Punctuation, u':'), (Token.Text, u'\n'), (Token.Text, u'            '), (Token.Name.Builtin.Pseudo, u'self'), (Token.Operator, u'.'), (Token.Name, u'metrics'), (Token.Punctuation, u'['), (Token.Literal.String, u"'"), (Token.Literal.String, u'mccabe'), (Token.Literal.String, u"'"), (Token.Punctuation, u']'), (Token.Text, u' '), (Token.Operator, u'+'), (Token.Operator, u'='), (Token.Text, u' '), (Token.Literal.Number.Integer, u'1'), (Token.Text, u'\n'), (Token.Text, u'        '), (Token.Text, u'\n')]

        lex = PythonLexer()
        tokenList = lex.get_tokens(code)
        self.assertEqual(list(tokenList), result)

class PythonTest(unittest.TestCase):
    def setUp(self):
        self.lexer = PythonLexer()

    def test_cls_builtin(self):
        """
        Tests that a cls token gets interpreted as a Token.Name.Builtin.Pseudo

        """
        fragment = 'class TestClass():\n    @classmethod\n    def hello(cls):\n        pass\n'
        tokens = [
            (Token.Keyword, 'class'),
            (Token.Text, ' '),
            (Token.Name.Class, 'TestClass'),
            (Token.Punctuation, '('),
            (Token.Punctuation, ')'),
            (Token.Punctuation, ':'),
            (Token.Text, '\n'),
            (Token.Text, '    '),
            (Token.Name.Decorator, '@classmethod'),
            (Token.Text, '\n'),
            (Token.Text, '    '),
            (Token.Keyword, 'def'),
            (Token.Text, ' '),
            (Token.Name.Function, 'hello'),
            (Token.Punctuation, '('),
            (Token.Name.Builtin.Pseudo, 'cls'),
            (Token.Punctuation, ')'),
            (Token.Punctuation, ':'),
            (Token.Text, '\n'),
            (Token.Text, '        '),
            (Token.Keyword, 'pass'),
            (Token.Text, '\n'),
        ]
        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))

 def __init__(self, makecolors = True, style = standardcols):
     if makecolors:
         self.makecolorpairs()
     if style is None:
         style = standardcols
     self.style = style
     self.lexer = PythonLexer()

 def get_tokens_unprocessed(self, text):
     for index, token, value in \
             PythonLexer.get_tokens_unprocessed(self, text):
         if token is Name and value in self.EXTRA_KEYWORDS:
             yield index, Keyword.Pseudo, value
         else:
             yield index, token, value

 def __init__(self):
     """ constructor """
     self._lexer = PythonLexer()
     self._formatter = HtmlFormatter()
     self._document = QtGui.QTextDocument()
     self._document.setDefaultStyleSheet(self._formatter.get_style_defs())
     self._format_cache = dict()