Python token.Text方法代码示例

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Text [as 别名]
def content_callback(self, match):
        content_type = getattr(self, "content_type", None)
        content = match.group()
        offset = match.start()
        if content_type:
            from pygments.lexers import get_lexer_for_mimetype

            try:
                lexer = get_lexer_for_mimetype(content_type)
            except ClassNotFound:
                pass
            else:
                for idx, token, value in lexer.get_tokens_unprocessed(content):
                    yield offset + idx, token, value
                return
        yield offset, Text, content 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Text [as 别名]
def get_background_style_defs(self, arg=None):
        prefix = self.get_css_prefix(arg)
        bg_color = self.style.background_color
        hl_color = self.style.highlight_color

        lines = []

        if arg and not self.nobackground and bg_color is not None:
            text_style = ''
            if Text in self.ttype2class:
                text_style = ' ' + self.class2style[self.ttype2class[Text]][0]
            lines.insert(
                0, '%s{ background: %s;%s }' % (
                    prefix(''), bg_color, text_style
                )
            )
        if hl_color is not None:
            lines.insert(
                0, '%s { background-color: %s }' % (prefix('hll'), hl_color)
            )

        return lines 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Text [as 别名]
def test_formatter_encodings():
    from pygments.formatters import HtmlFormatter

    # unicode output
    fmt = HtmlFormatter()
    tokens = [(Text, u"ä")]
    out = format(tokens, fmt)
    assert type(out) is str
    assert u"ä" in out

    # encoding option
    fmt = HtmlFormatter(encoding="latin1")
    tokens = [(Text, u"ä")]
    assert u"ä".encode("latin1") in format(tokens, fmt)

    # encoding and outencoding option
    fmt = HtmlFormatter(encoding="latin1", outencoding="utf8")
    tokens = [(Text, u"ä")]
    assert u"ä".encode("utf8") in format(tokens, fmt) 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Text [as 别名]
def test_can_cope_with_destructuring(lexer):
    fragment = u'val (a, b) = '
    tokens = [
        (Keyword, u'val'),
        (Text, u' '),
        (Punctuation, u'('),
        (Name.Property, u'a'),
        (Punctuation, u','),
        (Text, u' '),
        (Name.Property, u'b'),
        (Punctuation, u')'),
        (Text, u' '),
        (Punctuation, u'='),
        (Text, u' '),
        (Text, u'\n')
    ]
    assert list(lexer.get_tokens(fragment)) == tokens 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Text [as 别名]
def test_gcd_expr(lexer):
    fragment = u'1^3+(5-5)*gcd(a,b)\n'
    tokens = [
        (Token.Number.Integer, u'1'),
        (Token.Operator, u'^'),
        (Token.Literal.Number.Integer, u'3'),
        (Token.Operator, u'+'),
        (Token.Punctuation, u'('),
        (Token.Literal.Number.Integer, u'5'),
        (Token.Operator, u'-'),
        (Token.Literal.Number.Integer, u'5'),
        (Token.Punctuation, u')'),
        (Token.Operator, u'*'),
        (Token.Name, u'gcd'),
        (Token.Punctuation, u'('),
        (Token.Name, u'a'),
        (Token.Operator, u','),
        (Token.Name, u'b'),
        (Token.Punctuation, u')'),
        (Token.Text, u'\n')
    ]
    assert list(lexer.get_tokens(fragment)) == tokens 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Text [as 别名]
def test_escaped_bracestring(lexer):
    fragment = u'str.gsub(%r{\\\\\\\\}, "/")\n'
    tokens = [
        (Token.Name, u'str'),
        (Token.Operator, u'.'),
        (Token.Name, u'gsub'),
        (Token.Punctuation, u'('),
        (Token.Literal.String.Regex, u'%r{'),
        (Token.Literal.String.Regex, u'\\\\'),
        (Token.Literal.String.Regex, u'\\\\'),
        (Token.Literal.String.Regex, u'}'),
        (Token.Punctuation, u','),
        (Token.Text, u' '),
        (Token.Literal.String.Double, u'"'),
        (Token.Literal.String.Double, u'/'),
        (Token.Literal.String.Double, u'"'),
        (Token.Punctuation, u')'),
        (Token.Text, u'\n'),
    ]
    assert list(lexer.get_tokens(fragment)) == tokens 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Text [as 别名]
def test_enhanced_for(lexer):
    fragment = u'label:\nfor(String var2: var1) {}\n'
    tokens = [
        (Name.Label, u'label:'),
        (Text, u'\n'),
        (Keyword, u'for'),
        (Punctuation, u'('),
        (Name, u'String'),
        (Text, u' '),
        (Name, u'var2'),
        (Punctuation, u':'),
        (Text, u' '),
        (Name, u'var1'),
        (Punctuation, u')'),
        (Text, u' '),
        (Punctuation, u'{'),
        (Punctuation, u'}'),
        (Text, u'\n'),
    ]
    assert list(lexer.get_tokens(fragment)) == tokens 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Text [as 别名]
def test_unquoted_querystring(self):
        self.assertEqual(self.get_tokens('`echo name`==john'), [
            (Text, '`'),
            (Name.Builtin, 'echo'),
            (Text, 'name'),
            (Text, '`'),
            (Operator, '=='),
            (String, 'john')
        ])
        self.assertEqual(self.get_tokens('name==`echo john`'), [
            (Name, 'name'),
            (Operator, '=='),
            (Text, '`'),
            (Name.Builtin, 'echo'),
            (Text, 'john'),
            (Text, '`')
        ]) 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Text [as 别名]
def test_unquoted_bodystring(self):
        self.assertEqual(self.get_tokens('`echo name`=john'), [
            (Text, '`'),
            (Name.Builtin, 'echo'),
            (Text, 'name'),
            (Text, '`'),
            (Operator, '='),
            (String, 'john')
        ])
        self.assertEqual(self.get_tokens('name=`echo john`'), [
            (Name, 'name'),
            (Operator, '='),
            (Text, '`'),
            (Name.Builtin, 'echo'),
            (Text, 'john'),
            (Text, '`')
        ]) 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Text [as 别名]
def doctree_read(app, doctree):
    env = app.builder.env
    for node in doctree.traverse(addnodes.productionlist):
        for production in node:
            if not isinstance(production, addnodes.production):
                continue
            if not isinstance(production[-1], nodes.Text):
                continue
            parts = comment_re.split(production.pop().astext())
            new_nodes = []
            for s in parts:
                if comment_re.match(s):
                    new_nodes.append(nodes.emphasis(s, s))
                elif s:
                    new_nodes.append(nodes.Text(s))
            production += new_nodes 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Text [as 别名]
def test_skip_tokens_text(self):
        tokens = [(token.Text, "\n"), (token.Name, "foo")]
        result, skipped = self.tokenizer.skip_tokens(tokens)
        self.assertEqual(result, tokens[1:])
        self.assertEqual(skipped, 1) 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Text [as 别名]
def _should_skip(self, tok: RawToken) -> bool:
        if self.skip_text and tok[0] in token.Text:
            return True
        if self.skip_comments and tok[0] in token.Comment:
            return True
        return False 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Text [as 别名]
def header_callback(self, match):
        if match.group(1).lower() == 'content-type':
            content_type = match.group(5).strip()
            if ';' in content_type:
                content_type = content_type[:content_type.find(';')].strip()
            self.content_type = content_type
        yield match.start(1), Name.Attribute, match.group(1)
        yield match.start(2), Text, match.group(2)
        yield match.start(3), Operator, match.group(3)
        yield match.start(4), Text, match.group(4)
        yield match.start(5), Literal, match.group(5)
        yield match.start(6), Text, match.group(6) 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Text [as 别名]
def continuous_header_callback(self, match):
        yield match.start(1), Text, match.group(1)
        yield match.start(2), Literal, match.group(2)
        yield match.start(3), Text, match.group(3) 

# 需要导入模块: from pygments import token [as 别名]
# 或者: from pygments.token import Text [as 别名]
def content_callback(self, match):
        content_type = getattr(self, 'content_type', None)
        content = match.group()
        offset = match.start()
        if content_type:
            from pygments.lexers import get_lexer_for_mimetype
            try:
                lexer = get_lexer_for_mimetype(content_type)
            except ClassNotFound:
                pass
            else:
                for idx, token, value in lexer.get_tokens_unprocessed(content):
                    yield offset + idx, token, value
                return
        yield offset, Text, content