Java Token类代码示例

import org.antlr.v4.runtime.Token; //导入依赖的package包/类
@Override
protected void reportNoViableAlternative(Parser recognizer, NoViableAltException e) {
	// change error message from default implementation
	TokenStream tokens = recognizer.getInputStream();
	String input;
	if (tokens != null) {
		if (e.getStartToken().getType() == Token.EOF) {
			input = "the end";
		} else {
			input = escapeWSAndQuote(tokens.getText(e.getStartToken(), e.getOffendingToken()));
		}
	} else {
		input = escapeWSAndQuote("<unknown input>");
	}
	String msg = "inadmissible input at " + input;
	recognizer.notifyErrorListeners(e.getOffendingToken(), msg, e);
}
 

import org.antlr.v4.runtime.Token; //导入依赖的package包/类
@Override
protected String getTokenErrorDisplay(Token t) {
	// overwrite standard behavior to use "the end" instead of <EOF>
	if (t == null) {
		return "<no token>";
	}
	String s = getSymbolText(t).replace("<EOF>", "the end");
	if (s == null) {
		if (getSymbolType(t) == Token.EOF) {
			s = "the end";
		} else {
			s = escapeWSAndQuote("<" + getSymbolType(t) + ">");
		}
	}
	return s;
}
 

import org.antlr.v4.runtime.Token; //导入依赖的package包/类
@Override
public Token nextToken() {
    if (stashedNext != null) {
        previous = stashedNext;
        stashedNext = null;
        return previous;
    }
    Token next = super.nextToken();
    if (insertSemicolon(previous, next)) {
        stashedNext = next;
        previous = _factory.create(new Pair<TokenSource, CharStream>(this, _input), PainlessLexer.SEMICOLON, ";",
                Lexer.DEFAULT_TOKEN_CHANNEL, next.getStartIndex(), next.getStopIndex(), next.getLine(), next.getCharPositionInLine());
        return previous;
    } else {
        previous = next;
        return next;
    }
}
 

import org.antlr.v4.runtime.Token; //导入依赖的package包/类
@Override
protected boolean slashIsRegex() {
    Token lastToken = getPreviousToken();
    if (lastToken == null) {
        return true;
    }
    switch (lastToken.getType()) {
    case PainlessLexer.RBRACE:
    case PainlessLexer.RP:
    case PainlessLexer.OCTAL:
    case PainlessLexer.HEX:
    case PainlessLexer.INTEGER:
    case PainlessLexer.DECIMAL:
    case PainlessLexer.ID:
    case PainlessLexer.DOTINTEGER:
    case PainlessLexer.DOTID:
        return false;
    default:
        return true;
    }
}
 

import org.antlr.v4.runtime.Token; //导入依赖的package包/类
@Override
public void recover(final Parser recognizer, final RecognitionException re) {
    final Token token = re.getOffendingToken();
    String message;

    if (token == null) {
        message = "no parse token found.";
    } else if (re instanceof InputMismatchException) {
        message = "unexpected token [" + getTokenErrorDisplay(token) + "]" +
                " was expecting one of [" + re.getExpectedTokens().toString(recognizer.getVocabulary()) + "].";
    } else if (re instanceof NoViableAltException) {
        if (token.getType() == PainlessParser.EOF) {
            message = "unexpected end of script.";
        } else {
            message = "invalid sequence of tokens near [" + getTokenErrorDisplay(token) + "].";
        }
    } else {
        message =  "unexpected token near [" + getTokenErrorDisplay(token) + "].";
    }

    Location location = new Location(sourceName, token == null ? -1 : token.getStartIndex());
    throw location.createError(new IllegalArgumentException(message, re));
}
 

import org.antlr.v4.runtime.Token; //导入依赖的package包/类
/**
 * Make sure we don't attempt to recover inline; if the parser successfully
 * recovers, it won't throw an exception.
 */
@Override
public Token recoverInline(Parser recognizer) throws RecognitionException {
    InputMismatchException e = new InputMismatchException(recognizer);

    String policies = recognizer.getInputStream().getText();
    StringTokenizer tk = new StringTokenizer(policies, ";");
    String policy = "";
    int idx = 0;
    while (tk.hasMoreElements()) {
        policy = (String) tk.nextElement();
        idx += policy.length();
        if (idx >= e.getOffendingToken().getStartIndex()) {
            break;
        }
    }

    String message = Messages.get(Messages.DEFAULT_LOCALE,
            "error_invalid_firewallconfig", new Object[] {
                    e.getOffendingToken().getText(), policy });
    throw new RuntimeException(message);
}
 

import org.antlr.v4.runtime.Token; //导入依赖的package包/类
/**
 "If an operator has whitespace on the right side only, it is treated as a
 postfix unary operator. As an example, the ++ operator in a++ b is treated
 as a postfix unary operator."
 "If an operator has no whitespace on the left but is followed immediately
 by a dot (.), it is treated as a postfix unary operator. As an example,
 the ++ operator in a++.b is treated as a postfix unary operator (a++ .b
 rather than a ++ .b)."
 */
public static boolean isPostfixOp(TokenStream tokens) {
	int stop = getLastOpTokenIndex(tokens);
	if ( stop==-1 ) return false;

	int start = tokens.index();
	Token prevToken = tokens.get(start-1); // includes hidden-channel tokens
	Token nextToken = tokens.get(stop+1);
	boolean prevIsWS = isLeftOperatorWS(prevToken);
	boolean nextIsWS = isRightOperatorWS(nextToken);
	boolean result =
		!prevIsWS && nextIsWS ||
		!prevIsWS && nextToken.getType()==SwiftParser.DOT;
	String text = tokens.getText(Interval.of(start, stop));
	//System.out.println("isPostfixOp: '"+prevToken+"','"+text+"','"+nextToken+"' is "+result);
	return result;
}
 

import org.antlr.v4.runtime.Token; //导入依赖的package包/类
@Override
public Object visitTypedElementTuple(RAMLParser.TypedElementTupleContext typedeElementTuple) {
    final Token type = typedeElementTuple.type;
    final String name = typedeElementTuple.name.getText();

    final EObject propertyType = Strings.isNullOrEmpty(type.getText()) ?
            scope.getEObjectByName(BuiltinType.STRING.getName()) :
            typeExpressionConstructor.parse(type.getText(), scope);
    final boolean isRequired = !name.endsWith("?");
    scope.setValue(TYPED_ELEMENT__REQUIRED, isRequired, typedeElementTuple.getStart());
    final String parsedName = isRequired ? name : name.substring(0, name.length() - 1);

    scope.setValue(IDENTIFIABLE_ELEMENT__NAME, parsedName, typedeElementTuple.getStart());
    scope.setValue(TYPED_ELEMENT__TYPE, propertyType, typedeElementTuple.getStart());

    return scope.eObject();
}
 

import org.antlr.v4.runtime.Token; //导入依赖的package包/类
private EObject resolveType(final ParserRuleContext ruleContext, final EObject superType) {
    final EObject resolvedType;

    if (superType.eIsProxy()) {
        resolvedType = null;
    } else {
        resolvedType = create(superType.eClass(), ruleContext);
        EcoreUtil.replace(unresolved, resolvedType);

        final Token nameToken = ruleContext.getStart();
        final String name = nameToken.getText();
        final Scope typeScope = scope.with(resolvedType, TYPE_CONTAINER__TYPES);

        typeScope.with(IDENTIFIABLE_ELEMENT__NAME)
                .setValue(name, nameToken);

        typeScope.with(unresolved.eClass().getEStructuralFeature("type"))
                .setValue(superType, nameToken);
    }

    return resolvedType;
}
 

import org.antlr.v4.runtime.Token; //导入依赖的package包/类
public static boolean isClassName(TokenStream _input) {
  try {
    int i=1;
    Token token = _input.LT(i);
    while (token!=null && i < _input.size() && _input.LT(i+1).getType() == GroovyParser.DOT) {
      i = i + 2;
      token = _input.LT(i);
    }
    if(token==null)
      return false;
    // TODO here
    return Character.isUpperCase(Character.codePointAt(token.getText(), 0));
  } catch(Exception e) {
    e.printStackTrace();
  }
  
  return false;
}
 

import org.antlr.v4.runtime.Token; //导入依赖的package包/类
public StatementSplitter(String sql, Set<String> delimiters)
{
    TokenSource tokens = getLexer(sql, delimiters);
    ImmutableList.Builder<Statement> list = ImmutableList.builder();
    StringBuilder sb = new StringBuilder();
    while (true) {
        Token token = tokens.nextToken();
        if (token.getType() == Token.EOF) {
            break;
        }
        if (token.getType() == SqlBaseParser.DELIMITER) {
            String statement = sb.toString().trim();
            if (!statement.isEmpty()) {
                list.add(new Statement(statement, token.getText()));
            }
            sb = new StringBuilder();
        }
        else {
            sb.append(token.getText());
        }
    }
    this.completeStatements = list.build();
    this.partialStatement = sb.toString().trim();
}
 

import org.antlr.v4.runtime.Token; //导入依赖的package包/类
public static String squeezeStatement(String sql)
{
    TokenSource tokens = getLexer(sql, ImmutableSet.of());
    StringBuilder sb = new StringBuilder();
    while (true) {
        Token token = tokens.nextToken();
        if (token.getType() == Token.EOF) {
            break;
        }
        if (token.getType() == SqlBaseLexer.WS) {
            sb.append(' ');
        }
        else {
            sb.append(token.getText());
        }
    }
    return sb.toString().trim();
}
 

import org.antlr.v4.runtime.Token; //导入依赖的package包/类
@Override
public void exitNonReserved(SqlBaseParser.NonReservedContext context)
{
    // we can't modify the tree during rule enter/exit event handling unless we're dealing with a terminal.
    // Otherwise, ANTLR gets confused an fires spurious notifications.
    if (!(context.getChild(0) instanceof TerminalNode)) {
        int rule = ((ParserRuleContext) context.getChild(0)).getRuleIndex();
        throw new AssertionError("nonReserved can only contain tokens. Found nested rule: " + ruleNames.get(rule));
    }

    // replace nonReserved words with IDENT tokens
    context.getParent().removeLastChild();

    Token token = (Token) context.getChild(0).getPayload();
    context.getParent().addChild(new CommonToken(
            new Pair<>(token.getTokenSource(), token.getInputStream()),
            SqlBaseLexer.IDENTIFIER,
            token.getChannel(),
            token.getStartIndex(),
            token.getStopIndex()));
}
 

import org.antlr.v4.runtime.Token; //导入依赖的package包/类
private static ArithmeticBinaryExpression.Type getArithmeticBinaryOperator(Token operator)
{
    switch (operator.getType()) {
        case SqlBaseLexer.PLUS:
            return ArithmeticBinaryExpression.Type.ADD;
        case SqlBaseLexer.MINUS:
            return ArithmeticBinaryExpression.Type.SUBTRACT;
        case SqlBaseLexer.ASTERISK:
            return ArithmeticBinaryExpression.Type.MULTIPLY;
        case SqlBaseLexer.SLASH:
            return ArithmeticBinaryExpression.Type.DIVIDE;
        case SqlBaseLexer.PERCENT:
            return ArithmeticBinaryExpression.Type.MODULUS;
    }

    throw new UnsupportedOperationException("Unsupported operator: " + operator.getText());
}
 

import org.antlr.v4.runtime.Token; //导入依赖的package包/类
private static ComparisonExpressionType getComparisonOperator(Token symbol)
{
    switch (symbol.getType()) {
        case SqlBaseLexer.EQ:
            return ComparisonExpressionType.EQUAL;
        case SqlBaseLexer.NEQ:
            return ComparisonExpressionType.NOT_EQUAL;
        case SqlBaseLexer.LT:
            return ComparisonExpressionType.LESS_THAN;
        case SqlBaseLexer.LTE:
            return ComparisonExpressionType.LESS_THAN_OR_EQUAL;
        case SqlBaseLexer.GT:
            return ComparisonExpressionType.GREATER_THAN;
        case SqlBaseLexer.GTE:
            return ComparisonExpressionType.GREATER_THAN_OR_EQUAL;
    }

    throw new IllegalArgumentException("Unsupported operator: " + symbol.getText());
}