Java TokenStream类代码示例

import com.mitchellbosecke.pebble.lexer.TokenStream; //导入依赖的package包/类
private PebbleTemplate getPebbleTemplate(final PebbleEngine self, final String templateName, final Object cacheKey) throws LoaderException, ParserException {
    LexerImpl lexer = new LexerImpl(syntax, extensionRegistry.getUnaryOperators().values(),
            extensionRegistry.getBinaryOperators().values());
    Reader templateReader = self.retrieveReaderFromLoader(self.loader, cacheKey);
    TokenStream tokenStream = lexer.tokenize(templateReader, templateName);

    Parser parser = new ParserImpl(extensionRegistry.getUnaryOperators(),
            extensionRegistry.getBinaryOperators(), extensionRegistry.getTokenParsers());
    RootNode root = parser.parse(tokenStream);

    PebbleTemplateImpl instance = new PebbleTemplateImpl(self, root, templateName);

    for (NodeVisitorFactory visitorFactory : extensionRegistry.getNodeVisitors()) {
        visitorFactory.createVisitor(instance).visit(root);
    }

    return instance;
}
 

import com.mitchellbosecke.pebble.lexer.TokenStream; //导入依赖的package包/类
@Override
public RenderableNode parse(Token token, Parser parser) throws ParserException {
    TokenStream stream = parser.getStream();
    int lineNumber = token.getLineNumber();

    // skip the 'parallel' token
    stream.next();

    stream.expect(Token.Type.EXECUTE_END);

    BodyNode body = parser.subparse(decideParallelEnd);

    // skip the 'endparallel' token
    stream.next();

    stream.expect(Token.Type.EXECUTE_END);
    return new ParallelNode(lineNumber, body);
}
 

import com.mitchellbosecke.pebble.lexer.TokenStream; //导入依赖的package包/类
@Override
public RenderableNode parse(Token token, Parser parser) throws ParserException {
    TokenStream stream = parser.getStream();
    int lineNumber = token.getLineNumber();

    // skip over the tag name token
    stream.next();

    // parameter expressions will be added here
    Map<String, Expression<?>> paramExpressionMap = parseParams(stream, parser);
    Expression<?> tagBodyExpression = null;
    if(hasContent) {
        tagBodyExpression = parseBody(stream, parser);
    }
    else {
        stream.expect(Token.Type.EXECUTE_END);
    }

    return new TemplateTagNode(lineNumber, paramExpressionMap, tagBodyExpression);
}
 

import com.mitchellbosecke.pebble.lexer.TokenStream; //导入依赖的package包/类
public FilterInvocationExpression parseFilterInvocationExpression() throws ParserException {
    TokenStream stream = parser.getStream();
    Token filterToken = stream.expect(Token.Type.NAME);

    ArgumentsNode args = null;
    if (stream.current().test(Token.Type.PUNCTUATION, "(")) {
        args = this.parseArguments();
    } else {
        args = new ArgumentsNode(null, null, filterToken.getLineNumber());
    }

    return new FilterInvocationExpression(filterToken.getValue(), args, filterToken.getLineNumber());
}
 

import com.mitchellbosecke.pebble.lexer.TokenStream; //导入依赖的package包/类
private Expression<?> parseTestInvocationExpression() throws ParserException {
    TokenStream stream = parser.getStream();
    int lineNumber = stream.current().getLineNumber();

    Token testToken = stream.expect(Token.Type.NAME);

    ArgumentsNode args = null;
    if (stream.current().test(Token.Type.PUNCTUATION, "(")) {
        args = this.parseArguments();
    } else {
        args = new ArgumentsNode(null, null, testToken.getLineNumber());
    }

    return new TestInvocationExpression(lineNumber, testToken.getValue(), args);
}
 

import com.mitchellbosecke.pebble.lexer.TokenStream; //导入依赖的package包/类
/**
 * A bean attribute expression can either be an expression getting an
 * attribute from a variable in the context, or calling a method from a
 * variable.
 *
 * Ex. foo.bar or foo['bar'] or foo.bar('baz')
 *
 * @param node
 *            The expression parsed so far
 * @return NodeExpression The parsed subscript expression
 * @throws ParserException
 *             Thrown if a parsing error occurs.
 */
private Expression<?> parseBeanAttributeExpression(Expression<?> node) throws ParserException {
    TokenStream stream = parser.getStream();

    if (stream.current().test(Token.Type.PUNCTUATION, ".")) {

        // skip over the '.' token
        stream.next();

        Token token = stream.expect(Token.Type.NAME);

        ArgumentsNode args = null;
        if (stream.current().test(Token.Type.PUNCTUATION, "(")) {
            args = this.parseArguments();
            if (!args.getNamedArgs().isEmpty()) {
                throw new ParserException(null, "Can not use named arguments when calling a bean method", stream
                        .current().getLineNumber(), stream.getFilename());
            }
        }

        node = new GetAttributeExpression(node,
                new LiteralStringExpression(token.getValue(), token.getLineNumber()), args,
                stream.getFilename(), token.getLineNumber());

    } else if (stream.current().test(Token.Type.PUNCTUATION, "[")) {
        // skip over opening '[' bracket
        stream.next();

        node = new GetAttributeExpression(node, parseExpression(), stream.getFilename(), stream.current()
                .getLineNumber());

        // move past the closing ']' bracket
        stream.expect(Token.Type.PUNCTUATION, "]");
    }

    return node;
}
 

import com.mitchellbosecke.pebble.lexer.TokenStream; //导入依赖的package包/类
private Expression<?> parseArrayDefinitionExpression() throws ParserException {
    TokenStream stream = parser.getStream();

    // expect the opening bracket and check for an empty array
    stream.expect(Token.Type.PUNCTUATION, "[");
    if (stream.current().test(Token.Type.PUNCTUATION, "]")) {
        stream.next();
        return new ArrayExpression(stream.current().getLineNumber());
    }

    // there's at least one expression in the array
    List<Expression<?>> elements = new ArrayList<>();
    while (true) {
        Expression<?> expr = parseExpression();
        elements.add(expr);
        if (stream.current().test(Token.Type.PUNCTUATION, "]")) {
            // this seems to be the end of the array
            break;
        }
        // expect the comma separator, until we either find a closing
        // bracket or fail the expect
        stream.expect(Token.Type.PUNCTUATION, ",");
    }

    // expect the closing bracket
    stream.expect(Token.Type.PUNCTUATION, "]");

    return new ArrayExpression(elements, stream.current().getLineNumber());
}
 

import com.mitchellbosecke.pebble.lexer.TokenStream; //导入依赖的package包/类
private Expression<?> parseMapDefinitionExpression() throws ParserException {
    TokenStream stream = parser.getStream();

    // expect the opening brace and check for an empty map
    stream.expect(Token.Type.PUNCTUATION, "{");
    if (stream.current().test(Token.Type.PUNCTUATION, "}")) {
        stream.next();
        return new MapExpression(stream.current().getLineNumber());
    }

    // there's at least one expression in the map
    Map<Expression<?>, Expression<?>> elements = new HashMap<>();
    while (true) {
        // key : value
        Expression<?> keyExpr = parseExpression();
        stream.expect(Token.Type.PUNCTUATION, ":");
        Expression<?> valueExpr = parseExpression();
        elements.put(keyExpr, valueExpr);
        if (stream.current().test(Token.Type.PUNCTUATION, "}")) {
            // this seems to be the end of the map
            break;
        }
        // expect the comma separator, until we either find a closing brace
        // or fail the expect
        stream.expect(Token.Type.PUNCTUATION, ",");
    }

    // expect the closing brace
    stream.expect(Token.Type.PUNCTUATION, "}");

    return new MapExpression(elements, stream.current().getLineNumber());
}
 

import com.mitchellbosecke.pebble.lexer.TokenStream; //导入依赖的package包/类
@Override
public RenderableNode parse(Token token, Parser parser) throws ParserException {

    TokenStream stream = parser.getStream();
    int lineNumber = token.getLineNumber();

    // skip over the 'include' token
    stream.next();

    Expression<?> includeExpression = parser.getExpressionParser().parseExpression();

    Token current = stream.current();
    MapExpression mapExpression = null;

    // We check if there is an optional 'with' parameter on the include tag.
    if (current.getType().equals(Token.Type.NAME) && current.getValue().equals("with")) {

        // Skip over 'with'
        stream.next();

        Expression<?> parsedExpression = parser.getExpressionParser().parseExpression();

        if (parsedExpression instanceof MapExpression) {
            mapExpression = (MapExpression) parsedExpression;
        } else {
            throw new ParserException(null, String.format("Unexpected expression '%1s'.", parsedExpression
                    .getClass().getCanonicalName()), token.getLineNumber(), stream.getFilename());
        }

    }

    stream.expect(Token.Type.EXECUTE_END);

    return new IncludeNode(lineNumber, includeExpression, mapExpression);
}
 

import com.mitchellbosecke.pebble.lexer.TokenStream; //导入依赖的package包/类
@Override
public RenderableNode parse(Token token, Parser parser) throws ParserException {
    TokenStream stream = parser.getStream();
    int lineNumber = token.getLineNumber();

    // skip the 'extends' token
    stream.next();

    Expression<?> parentTemplateExpression = parser.getExpressionParser().parseExpression();

    stream.expect(Token.Type.EXECUTE_END);
    return new ExtendsNode(lineNumber, parentTemplateExpression);
}
 

import com.mitchellbosecke.pebble.lexer.TokenStream; //导入依赖的package包/类
@Override
public RenderableNode parse(Token token, Parser parser) throws ParserException {
    TokenStream stream = parser.getStream();
    int lineNumber = token.getLineNumber();

    // skip the 'filter' token
    stream.next();

    List<Expression<?>> filterInvocationExpressions = new ArrayList<>();

    filterInvocationExpressions.add(parser.getExpressionParser().parseFilterInvocationExpression());

    while(stream.current().test(Type.OPERATOR, "|")){
        // skip the '|' token
        stream.next();
        filterInvocationExpressions.add(parser.getExpressionParser().parseFilterInvocationExpression());
    }

    stream.expect(Token.Type.EXECUTE_END);

    BodyNode body = parser.subparse(endFilter);

    stream.next();
    stream.expect(Token.Type.EXECUTE_END);

    Expression<?> lastExpression = new RenderableNodeExpression(body, stream.current().getLineNumber());

    for(Expression<?> filterInvocationExpression : filterInvocationExpressions){

        FilterExpression filterExpression = new FilterExpression();
        filterExpression.setRight(filterInvocationExpression);
        filterExpression.setLeft(lastExpression);

        lastExpression = filterExpression;
    }

    return new PrintNode(lastExpression, lineNumber);
}
 

import com.mitchellbosecke.pebble.lexer.TokenStream; //导入依赖的package包/类
@Override
public String compile(String extension, String source, Map<String, Object> data) {
    try {
        LexerImpl lexer = new LexerImpl(
                engine.getSyntax(),
                engine.getExtensionRegistry().getUnaryOperators().values(),
                engine.getExtensionRegistry().getBinaryOperators().values());
        TokenStream tokenStream = lexer.tokenize(new StringReader(source), "");

        Parser parser = new ParserImpl(
                engine.getExtensionRegistry().getUnaryOperators(),
                engine.getExtensionRegistry().getBinaryOperators(),
                engine.getExtensionRegistry().getTokenParsers());
        RootNode root = parser.parse(tokenStream);

        PebbleTemplateImpl compiledTemplate = new PebbleTemplateImpl(engine, root, "");

        for (NodeVisitorFactory visitorFactory : engine.getExtensionRegistry().getNodeVisitors()) {
            visitorFactory.createVisitor(compiledTemplate).visit(root);
        }

        Writer writer = new StringWriter();
        compiledTemplate.evaluate(writer, data);

        return writer.toString();
    }
    catch (Exception e) {
        e.printStackTrace();
    }
    return source;
}
 

import com.mitchellbosecke.pebble.lexer.TokenStream; //导入依赖的package包/类
private Expression<?> parseBody(TokenStream stream, Parser parser) throws ParserException {
    List<Expression<?>> filterInvocationExpressions = new ArrayList<>();

    if(stream.current().test(Token.Type.PUNCTUATION, ":")) {
        stream.next();
        stream.expect(Token.Type.PUNCTUATION, ":");

        filterInvocationExpressions.add(parser.getExpressionParser().parseFilterInvocationExpression());

        while(stream.current().test(Token.Type.OPERATOR, "|")){
            // skip the '|' token
            stream.next();
            filterInvocationExpressions.add(parser.getExpressionParser().parseFilterInvocationExpression());
        }
    }

    stream.expect(Token.Type.EXECUTE_END);

    BodyNode body = parser.subparse(token -> token.test(Token.Type.NAME, "end" + name));

    stream.next();
    stream.expect(Token.Type.EXECUTE_END);

    Expression<?> lastExpression = new RenderableNodeExpression(body, stream.current().getLineNumber());

    for(Expression<?> filterInvocationExpression : filterInvocationExpressions){

        FilterExpression filterExpression = new FilterExpression();
        filterExpression.setRight(filterInvocationExpression);
        filterExpression.setLeft(lastExpression);

        lastExpression = filterExpression;
    }

    return lastExpression;
}
 

import com.mitchellbosecke.pebble.lexer.TokenStream; //导入依赖的package包/类
@Override
public TokenStream getStream() {
    return stream;
}
 

import com.mitchellbosecke.pebble.lexer.TokenStream; //导入依赖的package包/类
public void setStream(TokenStream stream) {
    this.stream = stream;
}