Java Token.getType方法代码示例

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
@Override
protected boolean slashIsRegex() {
    Token lastToken = getPreviousToken();
    if (lastToken == null) {
        return true;
    }
    switch (lastToken.getType()) {
    case PainlessLexer.RBRACE:
    case PainlessLexer.RP:
    case PainlessLexer.OCTAL:
    case PainlessLexer.HEX:
    case PainlessLexer.INTEGER:
    case PainlessLexer.DECIMAL:
    case PainlessLexer.ID:
    case PainlessLexer.DOTINTEGER:
    case PainlessLexer.DOTID:
        return false;
    default:
        return true;
    }
}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
@Override
public void recover(final Parser recognizer, final RecognitionException re) {
    final Token token = re.getOffendingToken();
    String message;

    if (token == null) {
        message = "no parse token found.";
    } else if (re instanceof InputMismatchException) {
        message = "unexpected token [" + getTokenErrorDisplay(token) + "]" +
                " was expecting one of [" + re.getExpectedTokens().toString(recognizer.getVocabulary()) + "].";
    } else if (re instanceof NoViableAltException) {
        if (token.getType() == PainlessParser.EOF) {
            message = "unexpected end of script.";
        } else {
            message = "invalid sequence of tokens near [" + getTokenErrorDisplay(token) + "].";
        }
    } else {
        message =  "unexpected token near [" + getTokenErrorDisplay(token) + "].";
    }

    Location location = new Location(sourceName, token == null ? -1 : token.getStartIndex());
    throw location.createError(new IllegalArgumentException(message, re));
}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
/**
 "If an operator has whitespace on the right side only, it is treated as a
 postfix unary operator. As an example, the ++ operator in a++ b is treated
 as a postfix unary operator."
 "If an operator has no whitespace on the left but is followed immediately
 by a dot (.), it is treated as a postfix unary operator. As an example,
 the ++ operator in a++.b is treated as a postfix unary operator (a++ .b
 rather than a ++ .b)."
 */
public static boolean isPostfixOp(TokenStream tokens) {
	int stop = getLastOpTokenIndex(tokens);
	if ( stop==-1 ) return false;

	int start = tokens.index();
	Token prevToken = tokens.get(start-1); // includes hidden-channel tokens
	Token nextToken = tokens.get(stop+1);
	boolean prevIsWS = isLeftOperatorWS(prevToken);
	boolean nextIsWS = isRightOperatorWS(nextToken);
	boolean result =
		!prevIsWS && nextIsWS ||
		!prevIsWS && nextToken.getType()==SwiftParser.DOT;
	String text = tokens.getText(Interval.of(start, stop));
	//System.out.println("isPostfixOp: '"+prevToken+"','"+text+"','"+nextToken+"' is "+result);
	return result;
}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
private static ArithmeticBinaryExpression.Type getArithmeticBinaryOperator(Token operator)
{
    switch (operator.getType()) {
        case SqlBaseLexer.PLUS:
            return ArithmeticBinaryExpression.Type.ADD;
        case SqlBaseLexer.MINUS:
            return ArithmeticBinaryExpression.Type.SUBTRACT;
        case SqlBaseLexer.ASTERISK:
            return ArithmeticBinaryExpression.Type.MULTIPLY;
        case SqlBaseLexer.SLASH:
            return ArithmeticBinaryExpression.Type.DIVIDE;
        case SqlBaseLexer.PERCENT:
            return ArithmeticBinaryExpression.Type.MODULUS;
    }

    throw new UnsupportedOperationException("Unsupported operator: " + operator.getText());
}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
private static ComparisonExpressionType getComparisonOperator(Token symbol)
{
    switch (symbol.getType()) {
        case SqlBaseLexer.EQ:
            return ComparisonExpressionType.EQUAL;
        case SqlBaseLexer.NEQ:
            return ComparisonExpressionType.NOT_EQUAL;
        case SqlBaseLexer.LT:
            return ComparisonExpressionType.LESS_THAN;
        case SqlBaseLexer.LTE:
            return ComparisonExpressionType.LESS_THAN_OR_EQUAL;
        case SqlBaseLexer.GT:
            return ComparisonExpressionType.GREATER_THAN;
        case SqlBaseLexer.GTE:
            return ComparisonExpressionType.GREATER_THAN_OR_EQUAL;
    }

    throw new IllegalArgumentException("Unsupported operator: " + symbol.getText());
}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
private static CurrentTime.Type getDateTimeFunctionType(Token token)
{
    switch (token.getType()) {
        case SqlBaseLexer.CURRENT_DATE:
            return CurrentTime.Type.DATE;
        case SqlBaseLexer.CURRENT_TIME:
            return CurrentTime.Type.TIME;
        case SqlBaseLexer.CURRENT_TIMESTAMP:
            return CurrentTime.Type.TIMESTAMP;
        case SqlBaseLexer.LOCALTIME:
            return CurrentTime.Type.LOCALTIME;
        case SqlBaseLexer.LOCALTIMESTAMP:
            return CurrentTime.Type.LOCALTIMESTAMP;
    }

    throw new IllegalArgumentException("Unsupported special function: " + token.getText());
}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
private static IntervalLiteral.IntervalField getIntervalFieldType(Token token)
{
    switch (token.getType()) {
        case SqlBaseLexer.YEAR:
            return IntervalLiteral.IntervalField.YEAR;
        case SqlBaseLexer.MONTH:
            return IntervalLiteral.IntervalField.MONTH;
        case SqlBaseLexer.DAY:
            return IntervalLiteral.IntervalField.DAY;
        case SqlBaseLexer.HOUR:
            return IntervalLiteral.IntervalField.HOUR;
        case SqlBaseLexer.MINUTE:
            return IntervalLiteral.IntervalField.MINUTE;
        case SqlBaseLexer.SECOND:
            return IntervalLiteral.IntervalField.SECOND;
    }

    throw new IllegalArgumentException("Unsupported interval field: " + token.getText());
}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
private void initIfFirstRun() {
  if (indentations.isEmpty()) {
    indentations.push(0);
    // First invocation. Look for the first non-NL. Enqueue it, and possibly an indentation if that non-NL
    // token doesn't start at char 0.
    Token firstRealToken;
    do {
      firstRealToken = pullToken();
    }
    while(firstRealToken.getType() == nlToken);

    if (firstRealToken.getCharPositionInLine() > 0) {
      indentations.push(firstRealToken.getCharPositionInLine());
      dentsBuffer.add(createToken(indentToken, firstRealToken));
    }
    dentsBuffer.add(firstRealToken);
  }
}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
private static ArithmeticBinaryExpression.Type getArithmeticBinaryOperator(Token operator) {
  switch (operator.getType()) {
    case SqlBaseLexer.PLUS:
      return ArithmeticBinaryExpression.Type.ADD;
    case SqlBaseLexer.MINUS:
      return ArithmeticBinaryExpression.Type.SUBTRACT;
    case SqlBaseLexer.ASTERISK:
      return ArithmeticBinaryExpression.Type.MULTIPLY;
    case SqlBaseLexer.SLASH:
      return ArithmeticBinaryExpression.Type.DIVIDE;
    case SqlBaseLexer.PERCENT:
      return ArithmeticBinaryExpression.Type.MODULUS;
    default:
      throw new UnsupportedOperationException("Unsupported operator: " + operator.getText());
  }
}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
public Token nextToken() {
  initIfFirstRun();
  Token t = dentsBuffer.isEmpty()
    ? pullToken()
    : dentsBuffer.remove();
  if (reachedEof) {
    return t;
  }
  final Token r;
  if (t.getType() == nlToken) {
    r = handleNewlineToken(t);
  } else if (t.getType() == Token.EOF) {
    r = eofHandler.apply(t);

  } else {
    r = t;
  }
  return r;
}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
public void computeFeatureVectorForToken(int i) {
	Token curToken = tokens.get(i);
	if ( curToken.getType()==Token.EOF ) return;

	int[] features = getFeatures(i);

	int injectNL_WS = getInjectWSCategory(tokens, i);

	int aligned = -1; // "don't care"
	if ( (injectNL_WS&0xFF)==CAT_INJECT_NL ) {
		TerminalNode node = tokenToNodeMap.get(curToken);
		aligned = getAlignmentCategory(doc, node, indentSize);
	}

	// track feature -> injectws, align decisions for token i
	corpus.addExemplar(doc, features, injectNL_WS, aligned);
}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
private static ComparisonExpression.Type getComparisonOperator(Token symbol) {
  switch (symbol.getType()) {
    case SqlBaseLexer.EQ:
      return ComparisonExpression.Type.EQUAL;
    case SqlBaseLexer.NEQ:
      return ComparisonExpression.Type.NOT_EQUAL;
    case SqlBaseLexer.LT:
      return ComparisonExpression.Type.LESS_THAN;
    case SqlBaseLexer.LTE:
      return ComparisonExpression.Type.LESS_THAN_OR_EQUAL;
    case SqlBaseLexer.GT:
      return ComparisonExpression.Type.GREATER_THAN;
    case SqlBaseLexer.GTE:
      return ComparisonExpression.Type.GREATER_THAN_OR_EQUAL;
    default:
      throw new IllegalArgumentException("Unsupported operator: " + symbol.getText());
  }
}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
private static boolean insertSemicolon(Token previous, Token next) {
    if (previous == null || next.getType() != PainlessLexer.RBRACK) {
        return false;
    }
    switch (previous.getType()) {
    case PainlessLexer.RBRACK:     // };} would be weird!
    case PainlessLexer.SEMICOLON:  // already have a semicolon, no need to add one
    case PainlessLexer.LBRACK:     // empty blocks don't need a semicolon
        return false;
    default:
        return true;
    }
}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
public void emit(Token token) {
    tokenIndex++;
    lastTokenType = token.getType();
    //System.out.println("EM: " + tokenNames[lastTokenType != -1 ? lastTokenType : 0] + ": " + lastTokenType + " TLE = " + (tlePos == tokenIndex) + " " + tlePos + "/" + tokenIndex + " " + token.getText());
    if (token.getType() == ROLLBACK_ONE) {
       ((PositionAdjustingLexerATNSimulator)getInterpreter()).resetAcceptPosition(getInputStream(), _tokenStartCharIndex - 1, _tokenStartLine, _tokenStartCharPositionInLine - 1);
    }
    super.emit(token);
}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
private BinaryFunctionExpr.Op binaryOperationFromToken(Token token) {
  switch (token.getType()) {
    case CellExpressionLexer.EQUALS:
      return BinaryFunctionExpr.Op.EQUALS;
    case CellExpressionLexer.NOT_EQUALS:
      return BinaryFunctionExpr.Op.NOT_EQUALS;
    case CellExpressionLexer.GREATER_THAN:
      return BinaryFunctionExpr.Op.GREATER_THAN;
    case CellExpressionLexer.GREATER_EQUALS:
      return BinaryFunctionExpr.Op.GREATER_EQUALS;
    case CellExpressionLexer.LESS_THAN:
      return BinaryFunctionExpr.Op.LESS_THAN;
    case CellExpressionLexer.LESS_EQUALS:
      return BinaryFunctionExpr.Op.LESS_EQUALS;
    case CellExpressionLexer.AND:
      return BinaryFunctionExpr.Op.AND;
    case CellExpressionLexer.OR:
      return BinaryFunctionExpr.Op.OR;
    case CellExpressionLexer.XOR:
      return BinaryFunctionExpr.Op.XOR;
    case CellExpressionLexer.PLUS:
      return BinaryFunctionExpr.Op.PLUS;
    case CellExpressionLexer.MINUS:
      return BinaryFunctionExpr.Op.MINUS;
    case CellExpressionLexer.MULT:
      return BinaryFunctionExpr.Op.MULTIPLICATION;
    case CellExpressionLexer.DIV:
      return BinaryFunctionExpr.Op.DIVISION;
    case CellExpressionLexer.MOD:
      return BinaryFunctionExpr.Op.MODULO;
    default:
      throw new ParseRuntimeException(
          new ParseException(token.getLine(), token.getCharPositionInLine(),
              "Unsupported singlesided operation: \"" + token.getType() + "\""));
  }
}