Java Token类代码示例

import jdk.nashorn.internal.parser.Token; //导入依赖的package包/类
/**
 * Dump a token stream to stdout
 *
 * TODO: most other bugging goes to stderr, change?
 *
 * @param source the source
 * @param lexer  the lexer
 * @param stream the stream to dump
 */
public static void dumpTokens(final Source source, final Lexer lexer, final TokenStream stream) {
    TokenType type;
    int k = 0;
    do {
        while (k > stream.last()) {
            // Get more tokens.
            lexer.lexify();
        }

        final long token = stream.get(k);
        type = Token.descType(token);
        System.out.println("" + k + ": " + Token.toString(source, token, true));
        k++;
    } while(type != EOF);
}
 

import jdk.nashorn.internal.parser.Token; //导入依赖的package包/类
@Override
public Node leaveExpressionStatement(final ExpressionStatement expressionStatement) {
    final Expression expr = expressionStatement.getExpression();
    ExpressionStatement node = expressionStatement;

    final FunctionNode currentFunction = lc.getCurrentFunction();

    if (currentFunction.isProgram()) {
        if (!isInternalExpression(expr) && !isEvalResultAssignment(expr)) {
            node = expressionStatement.setExpression(
                new BinaryNode(
                    Token.recast(
                        expressionStatement.getToken(),
                        TokenType.ASSIGN),
                    compilerConstant(RETURN),
                expr));
        }
    }

    return addStatement(node);
}
 

import jdk.nashorn.internal.parser.Token; //导入依赖的package包/类
@Override
public Node leaveBlock(final Block block) {
    if (!artificialBlock) {
        if (lc.isFunctionBody()) {
            // Prepend declaration-only var statements to the top of the statement list.
            lc.prependStatements(getCurrentFunctionState().varStatements);
        } else if (lc.isSplitBody()) {
            appendSplitReturn(FALLTHROUGH_STATE, NO_LINE_NUMBER);
            if (getCurrentFunctionState().fn.isProgram()) {
                // If we're splitting the program, make sure every shard ends with "return :return" and
                // begins with ":return = :return-in;".
                lc.prependStatement(new ExpressionStatement(NO_LINE_NUMBER, NO_TOKEN, NO_FINISH,
                        new BinaryNode(Token.toDesc(TokenType.ASSIGN, 0, 0), createReturnIdent(), createReturnParamIdent())));
            }
        }
    }
    return block;
}
 

import jdk.nashorn.internal.parser.Token; //导入依赖的package包/类
@Override
public Node leaveExpressionStatement(final ExpressionStatement expressionStatement) {
    final Expression expr = expressionStatement.getExpression();
    ExpressionStatement node = expressionStatement;

    final FunctionNode currentFunction = lc.getCurrentFunction();

    if (currentFunction.isProgram()) {
        if (!isInternalExpression(expr) && !isEvalResultAssignment(expr)) {
            node = expressionStatement.setExpression(
                new BinaryNode(
                    Token.recast(
                        expressionStatement.getToken(),
                        TokenType.ASSIGN),
                    compilerConstant(RETURN),
                expr));
        }
    }

    if (es6 && expressionStatement.destructuringDeclarationType() != null) {
        throwNotImplementedYet("es6.destructuring", expressionStatement);
    }

    return addStatement(node);
}
 

import jdk.nashorn.internal.parser.Token; //导入依赖的package包/类
/**
 * Extracts the comments tokens from a JavaScript source.
 *
 * @param aSource a source
 * @return a list of comment tokens
 */
public static List<Long> getCommentsTokens(String aSource) {
    TokenStream tokens = new TokenStream();
    Lexer lexer = new Lexer(Source.sourceFor("", aSource), tokens);//NOI18N
    long t;
    TokenType tt = TokenType.EOL;
    int i = 0;
    List<Long> commentsTokens = new ArrayList<>();
    while (tt != TokenType.EOF) {
        // Get next token in nashorn's parser way
        while (i > tokens.last()) {
            if (tokens.isFull()) {
                tokens.grow();
            }
            lexer.lexify();
        }
        t = tokens.get(i++);
        tt = Token.descType(t);
        if (tt == TokenType.COMMENT) {
            commentsTokens.add(t);
        }
    }
    return commentsTokens;
}
 

import jdk.nashorn.internal.parser.Token; //导入依赖的package包/类
/**
 * Removes all commentaries from some JavaScript code.
 *
 * @param text a source
 * @return comments-free JavaScript code
 */
public static String removeComments(String text) {
    StringBuilder sb = new StringBuilder();
    int i = 0;
    for (Long t : getCommentsTokens(text)) {
        int offset = Token.descPosition(t);
        int lenght = Token.descLength(t);
        sb.append(text.substring(i, offset));
        for (int j = 0; j < lenght; j++) {
            sb.append(" ");//NOI18N
        }
        i = offset + lenght;
    }
    sb.append(text.substring(i));
    return sb.toString();
}
 

import jdk.nashorn.internal.parser.Token; //导入依赖的package包/类
/**
 * Format an error message to include source and line information.
 * @param message Error message string.
 * @param source  Source file information.
 * @param line    Source line number.
 * @param column  Source column number.
 * @param token   Offending token descriptor.
 * @return formatted string
 */
public static String format(final String message, final Source source, final int line, final int column, final long token) {
    final String        eoln     = System.lineSeparator();
    final int           position = Token.descPosition(token);
    final StringBuilder sb       = new StringBuilder();

    // Source description and message.
    sb.append(source.getName()).
        append(':').
        append(line).
        append(':').
        append(column).
        append(' ').
        append(message).
        append(eoln);

    // Source content.
    final String sourceLine = source.getSourceLine(position);
    sb.append(sourceLine).append(eoln);

    // Pointer to column.
    for (int i = 0; i < column; i++) {
        if (sourceLine.charAt(i) == '\t') {
            sb.append('\t');
        } else {
            sb.append(' ');
        }
    }

    sb.append('^');
    // Use will append eoln.
    // buffer.append(eoln);

    return sb.toString();
}
 

import jdk.nashorn.internal.parser.Token; //导入依赖的package包/类
@Override
String toSource() {
    if (source != null && token != 0) {
        return source.getString(Token.descPosition(token), Token.descLength(token));
    }

    return "function " + (name == null ? "" : name) + "() { [native code] }";
}
 

import jdk.nashorn.internal.parser.Token; //导入依赖的package包/类
private static long tokenFor(final FunctionNode fn) {
    final int  position  = Token.descPosition(fn.getFirstToken());
    final long lastToken = Token.withDelimiter(fn.getLastToken());
    // EOL uses length field to store the line number
    final int  length    = Token.descPosition(lastToken) - position + (Token.descType(lastToken) == TokenType.EOL ? 0 : Token.descLength(lastToken));

    return Token.toDesc(TokenType.FUNCTION, position, length);
}
 

import jdk.nashorn.internal.parser.Token; //导入依赖的package包/类
FunctionNode reparse() {
    if (isSerialized()) {
        return deserialize();
    }

    final int descPosition = Token.descPosition(token);
    final Context context = Context.getContextTrusted();
    final Parser parser = new Parser(
        context.getEnv(),
        source,
        new Context.ThrowErrorManager(),
        isStrict(),
        // source starts at line 0, so even though lineNumber is the correct declaration line, back off
        // one to make it exclusive
        lineNumber - 1,
        context.getLogger(Parser.class));

    if (getFunctionFlag(FunctionNode.IS_ANONYMOUS)) {
        parser.setFunctionName(functionName);
    }
    parser.setReparsedFunction(this);

    final FunctionNode program = parser.parse(CompilerConstants.PROGRAM.symbolName(), descPosition,
            Token.descLength(token), true);
    // Parser generates a program AST even if we're recompiling a single function, so when we are only
    // recompiling a single function, extract it from the program.
    return (isProgram() ? program : extractFunctionFromScript(program)).setName(null, functionName);
}
 

import jdk.nashorn.internal.parser.Token; //导入依赖的package包/类
/**
 * Constructor
 *
 * @param token   token
 * @param finish  finish
 * @param value   array literal value, a Node array
 */
protected ArrayLiteralNode(final long token, final int finish, final Expression[] value) {
    super(Token.recast(token, TokenType.ARRAY), finish, value);
    this.elementType = Type.UNKNOWN;
    this.presets     = null;
    this.postsets    = null;
    this.units       = null;
}
 

import jdk.nashorn.internal.parser.Token; //导入依赖的package包/类
@Override
public void toString(final StringBuilder sb, final boolean printType) {
    sb.append(Token.descType(getToken()).getName()).append(' ');
    name.toString(sb, printType);

    if (init != null) {
        sb.append(" = ");
        init.toString(sb, printType);
    }
}
 

import jdk.nashorn.internal.parser.Token; //导入依赖的package包/类
private static IfNode makeIfStateEquals(final int lineNumber, final long token, final int finish,
        final int value, final Block pass, final Statement fail) {
    return new IfNode(lineNumber, token, finish,
            new BinaryNode(Token.recast(token, TokenType.EQ_STRICT),
                    GetSplitState.INSTANCE, intLiteral(value)),
            pass,
            fail == null ? null : new Block(NO_TOKEN, NO_FINISH, fail));
}
 

import jdk.nashorn.internal.parser.Token; //导入依赖的package包/类
/**
 * Format an error message to include source and line information.
 * @param message Error message string.
 * @param source  Source file information.
 * @param line    Source line number.
 * @param column  Source column number.
 * @param token   Offending token descriptor.
 * @return formatted string
 */
public static String format(final String message, final Source source, final int line, final int column, final long token) {
    final String        eoln     = System.lineSeparator();
    final int           position = Token.descPosition(token);
    final StringBuilder sb       = new StringBuilder();

    // Source description and message.
    sb.append(source.getName()).
        append(':').
        append(line).
        append(':').
        append(column).
        append(' ').
        append(message).
        append(eoln);

    // Source content.
    final String sourceLine = source.getSourceLine(position);
    sb.append(sourceLine).append(eoln);

    // Pointer to column.
    for (int i = 0; i < column; i++) {
        if (i < sourceLine.length() && sourceLine.charAt(i) == '\t') {
            sb.append('\t');
        } else {
            sb.append(' ');
        }
    }

    sb.append('^');
    // Use will append eoln.
    // buffer.append(eoln);

    return sb.toString();
}
 

import jdk.nashorn.internal.parser.Token; //导入依赖的package包/类
/**
 * Constructor
 *
 * @param token   token
 * @param finish  finish
 * @param value   array literal value, a Node array
 * @param hasSpread true if the array has a spread element
 * @param hasTrailingComma true if the array literal has a comma after the last element
 */
protected ArrayLiteralNode(final long token, final int finish, final Expression[] value, final boolean hasSpread, final boolean hasTrailingComma) {
    super(Token.recast(token, TokenType.ARRAY), finish, value);
    this.elementType = Type.UNKNOWN;
    this.presets     = null;
    this.postsets    = null;
    this.splitRanges = null;
    this.hasSpread        = hasSpread;
    this.hasTrailingComma = hasTrailingComma;
}