Python token.Name方法代码示例

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Name [as 别名]
def test_skip_tokens_comments(self):
        tokens = [(token.Comment.Multiline, "foo"), (token.Name, "foo")]
        result, skipped = self.tokenizer.skip_tokens(tokens)
        self.assertEqual(result, tokens[1:])
        self.assertEqual(skipped, 1) 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Name [as 别名]
def test_skip_tokens_text(self):
        tokens = [(token.Text, "\n"), (token.Name, "foo")]
        result, skipped = self.tokenizer.skip_tokens(tokens)
        self.assertEqual(result, tokens[1:])
        self.assertEqual(skipped, 1) 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Name [as 别名]
def test_tokenize(self):
        expected = [Token("Name", "foo"), Token("Operator", "&&"), Token("Name", "bar")]
        self.assertEqual(self.tokenizer.tokenize_string("foo && bar"), expected) 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Name [as 别名]
def analyze(self, text):
        if any([token is Error for token, value in self.get_tokens(text)]):
            return 2 * (None, )
        tokens, args, kwargs = self.get_tokens(text), [], {}
        for token, value in tokens:
            if token is Keyword:
                token = token in ['true', 'True']
            elif token is Number:
                token = int(token)
            if token in (Keyword, Number, String):
                args.append(value)
            if token is Name:
                next(tokens)  # pass the Operator '='
                kwargs.update({value: next(tokens)[1]})
        return args, kwargs 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Name [as 别名]
def test_can_reject_almost_float(lexer):
    _assert_tokens_match(lexer, '.e1', ((Punctuation, '.'), (Name, 'e1'))) 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Name [as 别名]
def test_can_lex_names(lexer):
    _assert_are_tokens_of_type(lexer,
                               u'thingy thingy123 _thingy _ _123 Ähnliches Müll #temp1 ##temp2',
                               Name) 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Name [as 别名]
def test_can_reject_almost_float(lexer):
    assert_tokens_match(lexer, '.e1', ((Punctuation, '.'), (Name, 'e1'))) 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Name [as 别名]
def test_can_lex_names(lexer):
    assert_are_tokens_of_type(lexer, u'thingy thingy123 _thingy _123', Name) 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Name [as 别名]
def test_can_recover_after_unterminated_string(lexer):
    assert_tokens_match(lexer,
                        '"x\nx',
                        ((String.Double, '"'), (String.Double, 'x'),
                         (Error, '\n'), (Name, 'x'))) 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Name [as 别名]
def get_tokens_unprocessed(self, text):
        for index, token, value in \
                PythonLexer.get_tokens_unprocessed(self, text):
            if token is Name and value in self.EXTRA_KEYWORDS:
                yield index, Keyword.Pseudo, value
            else:
                yield index, token, value 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Name [as 别名]
def get_parser(self):
        parser = argparse.ArgumentParser(
            'sos convert FILE.sos FILE.html (or --to html)',
            description='''Convert sos file to html format with syntax highlighting,
            and save the output either to a HTML file or view it in a broaser.'''
        )
        parser.add_argument(
            '--url',
            help='''URL to the raw sos file, which will be linked
            to filenames in the HTML output''')
        parser.add_argument('--raw', help=argparse.SUPPRESS)
        parser.add_argument(
            '--style',
            choices=codemirror_themes,
            help="Code mirror themes for the output",
            default='default')
        parser.add_argument(
            '--linenos', action='store_true', help=argparse.SUPPRESS)
        parser.add_argument(
            '--template',
            help='''Name or path to an alternative template for
                converting sos script to HTML.  The template can use variables
                "filename" for full name of the script as provided, "basename"
                as the name part of the filename, "script" for the content of
                the script, "sov_version" for version of sos, "linenos" as a
                flag to whether or not line numbers should be displayed, "url"
                as an optional URL for the script, and "theme" as provided by
                option --style.''')
        parser.add_argument(
            '--view',
            action='store_true',
            help='''Open the output file in a broswer. In case
            no html file is specified, this option will display the HTML file in
            a browser, instead of writing its content to standard output.''')
        return parser 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Name [as 别名]
def build_graph(file_paths, graph, lexer):
    """
    for each file in the set of files
        create a node and add it to the graph
        open the file
        read the contents into memory
        get a list of tokens from the lexer
        for each token in the resulting tokens
            check if the token is defining a symbol
            if true, add the symbol to the file node

    for each file in the set of files
        open the file
        read the contents into memory
        get a list of token from the lexer
        for each token in the resulting tokens
            check if the token is using a symbol
            if true:
                search the graph for the node that has the symbol definition
                create a relationship from the current file to the node with
                the symbol definition
    """
    for file_path in file_paths:
        node = Node(file_path)
        graph.add_node(node)
        try:
            with open(file_path, 'r', encoding='utf-8') as file:
                contents = file.read()

            tokens = lexer.get_tokens(contents)
            for item in tokens:
                token_type = item[0]
                symbol = item[1]
                if token_type in [token.Name.Function, token.Name.Class]:
                    node.defines.add(symbol)
                elif token_type in TOKENTYPE_WHITELIST:
                    node.references.add(symbol)
            if 'DEBUG' in os.environ:
                print(node)
        except FileNotFoundError as e:
            continue
        except UnicodeDecodeError:
            continue

    for caller in graph.nodes_iter():
        for reference in caller.references:
            for callee in graph.nodes_iter():
                if callee is not caller and reference in callee.defines:
                    graph.add_edge(caller, callee)