Java Token.INVALID_TOKEN_TYPE属性代码示例

@Override
public void consumeToken(Token token) {
    if (backtracking > 0 || resync) {
        return;
    }

    if (debug_tokens) {
        CommonToken ct = (CommonToken) token;
        int[] ctr = CommonTokenUtil.getCommonTokenOffsetRange(ct);
        System.out.println(token + "(" + ctr[0] + "-" + ctr[1] + ")");
    }

    //ignore the closing EOF token, we do not want it
    //it the parse tree
    if (token.getType() == Css3Lexer.EOF) {
        return;
    }

    //also ignore error tokens - they are added as children of ErrorNode-s in the recognitionException(...) method
    if (token.getType() == Token.INVALID_TOKEN_TYPE) {
        return;
    }

    lastConsumedToken = (CommonToken) token;

    RuleNode ruleNode = callStack.peek();
    TokenNode elementNode = new TokenNode(source, (CommonToken) token);
    elementNode.hiddenTokens = this.hiddenTokens;
    hiddenTokens.clear();
    ruleNode.addChild(elementNode);

    updateFirstTokens(ruleNode, lastConsumedToken);
}
 

private ILeafNode createLeafNode(Token token, EObject grammarElement) {
	boolean isHidden = token.getChannel() == HIDDEN;
	SyntaxErrorMessage error = null;
	if (!isHidden) {
		if (currentError != null) {
			error = currentError;
			currentError = null;
		}
	}
	if (token.getType() == Token.INVALID_TOKEN_TYPE) {
		if (error == null) {
			String lexerErrorMessage = ((XtextTokenStream) input).getLexerErrorMessage(token);
			LexerErrorContext errorContext = new LexerErrorContext(lexerErrorMessage);
			error = syntaxErrorProvider.getSyntaxErrorMessage(errorContext);
		}
	}
	if (grammarElement == null) {
		String ruleName = antlrTypeToLexerName.get(token.getType());
		grammarElement = allRules.get(ruleName);
	}
	CommonToken commonToken = (CommonToken) token;
	if (error != null)
		hadErrors = true;
	return nodeBuilder.newLeafNode(
			commonToken.getStartIndex(), 
			commonToken.getStopIndex() - commonToken.getStartIndex() + 1, 
			grammarElement, 
			isHidden, 
			error, 
			currentNode);
}
 

@Override
public Token nextToken() {
	while (true) {
		this.state.token = null;
		this.state.channel = Token.DEFAULT_CHANNEL;
		this.state.tokenStartCharIndex = input.index();
		this.state.tokenStartCharPositionInLine = input.getCharPositionInLine();
		this.state.tokenStartLine = input.getLine();
		this.state.text = null;
		if (input.LA(1) == CharStream.EOF) {
			return Token.EOF_TOKEN;
		}
		try {
			mTokens();
			if (this.state.token == null) {
				emit();
			}
			else if (this.state.token == Token.SKIP_TOKEN) {
				continue;
			}
			return this.state.token;
		}
		catch (RecognitionException re) {
			reportError(re);
			if (re instanceof NoViableAltException ||
				re instanceof FailedPredicateException) {
				recover(re);
			}
			// create token that holds mismatched char
			Token t = new CommonToken(input, Token.INVALID_TOKEN_TYPE, Token.HIDDEN_CHANNEL,
					this.state.tokenStartCharIndex, getCharIndex() - 1);
			t.setLine(this.state.tokenStartLine);
			t.setCharPositionInLine(this.state.tokenStartCharPositionInLine);
			tokenErrorMap.put(t, getErrorMessage(re, this.getTokenNames()));
			emit(t);
			return this.state.token;
		}
	}
}
 

protected String getMappedValue(final int tokenType) {
  String _xifexpression = null;
  if ((tokenType == Token.INVALID_TOKEN_TYPE)) {
    _xifexpression = HighlightingStyles.INVALID_TOKEN_ID;
  } else {
    _xifexpression = this.mappedValues[(tokenType - Token.MIN_TOKEN_TYPE)];
  }
  return _xifexpression;
}
 

@Override
public String getPartitionType(final int antlrTokenType) {
  // on lexer error return default content type
  if (antlrTokenType == Token.INVALID_TOKEN_TYPE) {
    return IDocument.DEFAULT_CONTENT_TYPE;
  }
  return getMappedValue(antlrTokenType);
}
 

public String getLexerErrorMessage(Token invalidToken) {
	if (tokenSource instanceof org.eclipse.xtext.parser.antlr.Lexer) {
		return ((org.eclipse.xtext.parser.antlr.Lexer) tokenSource).getErrorMessage(invalidToken);
	}
	return (invalidToken.getType() == Token.INVALID_TOKEN_TYPE) ? "Invalid token " + invalidToken.getText() : null;
}