Java Token.EOF属性代码示例

/** Consume tokens until one matches the given token set */
    @Override
    public void consumeUntil(IntStream i, BitSet set) {
//        System.out.println("consumeUntil(" + set.toString(getTokenNames()) + ")");
        Token ttype;
        List<Token> skipped = new ArrayList<>();
        beginResync();
        try {
            while ((ttype = input.LT(1)) != null && ttype.getType() != Token.EOF && !set.member(ttype.getType())) {
//            System.out.println("consume during recover LA(1)=" + getTokenNames()[input.LA(1)]);
                input.consume();
                skipped.add(ttype);
            }
        } finally {
            endResync();
        }
        ((NbParseTreeBuilder) dbg).consumeSkippedTokens(skipped);
    }
 

public String toTokenString() {
	CharStream input = this.token.getInputStream();
	GrammarASTAdaptor adaptor = new GrammarASTAdaptor(input);
	CommonTreeNodeStream nodes =
		new CommonTreeNodeStream(adaptor, this);
	StringBuilder buf = new StringBuilder();
	GrammarAST o = (GrammarAST)nodes.LT(1);
	int type = adaptor.getType(o);
	while ( type!=Token.EOF ) {
		buf.append(" ");
		buf.append(o.getText());
		nodes.consume();
		o = (GrammarAST)nodes.LT(1);
		type = adaptor.getType(o);
	}
	return buf.toString();
}
 

public Token nextToken()
{
    super.nextToken();
    if (tokens.size() == 0)
        return new CommonToken(Token.EOF);
    return tokens.remove(0);
}
 

/**
 * <p>
 * Promotes EOL which may lead to an automatically inserted semicolon. This is probably the most important method
 * for automatic semicolon insertion, as it is only possible to insert a semicolon in case of line breaks (even if
 * they are hidden in a multi-line comment!).
 * </p>
 */
public static void promoteEOL(Callback callback) {
	RecognizerSharedState state = callback.getState();
	TokenStream input = callback.getInput();
	// Don't promote EOL if there was a syntax error at EOF
	if (state.lastErrorIndex == input.size()) {
		return;
	}
	// Get current token and its type (the possibly offending token).
	Token prev = input.LT(-1);
	Token next = input.LT(1);
	int la = next.getType();

	// Promoting an EOL means switching it from off channel to on channel.
	// A ML_COMMENT gets promoted when it contains an EOL.
	for (int idx = prev == null ? 0 : prev.getTokenIndex() + 1, max = la == Token.EOF ? input.size()
			: next.getTokenIndex(); idx < max; idx++) {
		Token lt = input.get(idx);
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
			// On channel token found: stop scanning (previously promoted)
			break;
		} else if (isSemicolonEquivalent(lt)) {
			// We found our EOL: promote the token to on channel, position the input on it and reset the rule
			// start.
			lt.setChannel(Token.DEFAULT_CHANNEL);
			input.seek(idx);
			break;
		}
	}
}
 

protected boolean isSameTokenSequence(TokenSource originalSource, TokenSource newSource, int expectedLength) {
	Token token = originalSource.nextToken();
	int newLength = 0;
	while(Token.EOF != token.getType()) {
		Token newToken = newSource.nextToken();
		if (token.getType() != newToken.getType()) {
			return false;
		}
		newLength += TokenTool.getLength(newToken);
		token = originalSource.nextToken();
	}
	return newLength == expectedLength;
}
 

/** Add an EOF transition to any rule end ATNState that points to nothing
    *  (i.e., for all those rules not invoked by another rule).  These
    *  are start symbols then.
 *
 *  Return the number of grammar entry points; i.e., how many rules are
 *  not invoked by another rule (they can only be invoked from outside).
 *  These are the start rules.
    */
public int addEOFTransitionToStartRules() {
	int n = 0;
	ATNState eofTarget = newState(null); // one unique EOF target for all rules
	for (Rule r : g.rules.values()) {
		ATNState stop = atn.ruleToStopState[r.index];
		if ( stop.getNumberOfTransitions()>0 ) continue;
		n++;
		Transition t = new AtomTransition(eofTarget, Token.EOF);
		stop.addTransition(t);
	}
	return n;
}
 

@Override
protected void doSplitToken(Token token, ITokenAcceptor result) {
	if (token.getType() == Token.EOF) {
		doSplitEofToken(token, result);
	} else {
		doSplitTokenImpl(token, result);	
	}
}
 

@Test public void testConsumeUntilEOF_large() {
	tokenCount = 25000;
	XtextTokenStream stream = new XtextTokenStream(this, Token.DEFAULT_CHANNEL);
	while(stream.LA(1) != Token.EOF) {
		stream.consume();
		stream.LA(1);
	}
}
 

@Test public void testConsumeUntilEOF_small() {
	tokenCount = 25;
	XtextTokenStream stream = new XtextTokenStream(this, Token.DEFAULT_CHANNEL);
	while(stream.LA(1) != Token.EOF) {
		stream.consume();
		stream.LA(1);
	}
}
 

protected void doTestLookahead(int lookAhead, int tokenCount) {
	this.tokenCount = tokenCount;
	XtextTokenStream stream = new XtextTokenStream(this, Token.DEFAULT_CHANNEL);
	while(stream.LA(1) != Token.EOF) {
		for(int i = 0; i < lookAhead; i++) {
			for(int j = i + 1; j < lookAhead; j++) {
				stream.LA(j - i);					
			}
			stream.consume();
		}
	}
}
 

@Override
protected void doSplitToken(Token token, ITokenAcceptor result) {
	int nextLevel = (token.getType() == Token.EOF) ? 0 : countTabs(token.getText());
	while (this.level > nextLevel) {
		this.level--;
		result.accept(new CommonToken(InternalIndentationAwareTestLanguageLexer.RULE_END, ""));
	}
	result.accept(token);
	while (this.level < nextLevel) {
		this.level++;
		result.accept(new CommonToken(InternalIndentationAwareTestLanguageLexer.RULE_BEGIN, ""));
	}
}
 

@Override
public int LA(int i) {
	Token lookaheadToken = LT(i);
	int result = lookaheadToken.getType();
	if (result == Token.EOF && getListener() != null)
		getListener().announceEof(i);
	return result;
}
 

@Override
public void recover(IntStream stream, RecognitionException ex) {
	if (recoveryListener != null)
		recoveryListener.beginErrorRecovery();
	removeUnexpectedElements();
	if (ex instanceof FailedPredicateException && ex.token.getType() == Token.EOF) {
		failedPredicateAtEOF = true;
	}
	super.recover(stream, ex);
	if (recoveryListener != null)
		recoveryListener.endErrorRecovery();
}
 

@Override
protected boolean shouldSplitToken(Token token) {
	return token.getType() == Token.EOF || shouldSplitTokenImpl(token);
}
 

@Override
protected boolean shouldSplitToken(Token token) {
	return token.getType() == InternalIndentationAwareTestLanguageLexer.RULE_NL || token.getType() == Token.EOF;
}