Java Token.getTokenIndex方法代码示例

import org.antlr.runtime.Token; //导入方法依赖的package包/类
@Override
protected void announce(Token start, Token stop, AbstractElement element) {
	if (start != null && start != Token.EOF_TOKEN) {
		if (start == stop) {
			announce(start, element);
		} else {
			CommonToken castedStart = (CommonToken) start;
			if (stop == null) { // possible error condition
				if (start.getTokenIndex() == state.lastErrorIndex) {
					return;
				}
			}
			CommonToken castedEnd = (CommonToken) stop;
			Integer newType = rewriter.rewrite(castedStart, element);
			if (newType != null && castedEnd != null && castedEnd != Token.EOF_TOKEN) {
				LazyTokenStream castedInput = (LazyTokenStream) this.input;
				for (int i = castedStart.getTokenIndex() + 1; i < castedEnd.getTokenIndex(); i++) {
					Token token = castedInput.get(i);
					if (token.getChannel() != Token.HIDDEN_CHANNEL)
						token.setType(newType);
				}
				castedEnd.setType(newType);
			}
		}
	}
}
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
/**
 * Returns true if there was an unexpected EOL.
 */
public static boolean hasDisallowedEOL(Callback callback) {
	TokenStream input = callback.getInput();
	Token lt = input.LT(1);

	// Start on the position before the current token and scan backwards off channel tokens until the previous on
	// channel token.
	for (int ix = lt.getTokenIndex() - 1; ix > 0; ix--) {
		lt = input.get(ix);
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
			// On channel token found: stop scanning.
			break;
		} else if (isSemicolonEquivalent(lt)) {
			return true;
		}
	}
	return false;
}
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
/**
 * <p>
 * Promotes EOL which may lead to an automatically inserted semicolon. This is probably the most important method
 * for automatic semicolon insertion, as it is only possible to insert a semicolon in case of line breaks (even if
 * they are hidden in a multi-line comment!).
 * </p>
 */
public static void promoteEOL(Callback callback) {
	RecognizerSharedState state = callback.getState();
	TokenStream input = callback.getInput();
	// Don't promote EOL if there was a syntax error at EOF
	if (state.lastErrorIndex == input.size()) {
		return;
	}
	// Get current token and its type (the possibly offending token).
	Token prev = input.LT(-1);
	Token next = input.LT(1);
	int la = next.getType();

	// Promoting an EOL means switching it from off channel to on channel.
	// A ML_COMMENT gets promoted when it contains an EOL.
	for (int idx = prev == null ? 0 : prev.getTokenIndex() + 1, max = la == Token.EOF ? input.size()
			: next.getTokenIndex(); idx < max; idx++) {
		Token lt = input.get(idx);
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
			// On channel token found: stop scanning (previously promoted)
			break;
		} else if (isSemicolonEquivalent(lt)) {
			// We found our EOL: promote the token to on channel, position the input on it and reset the rule
			// start.
			lt.setChannel(Token.DEFAULT_CHANNEL);
			input.seek(idx);
			break;
		}
	}
}
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
/** Returns the token that comes first in the input stream. */
private Token getMin(Token one, Token two) {
    if (one.getTokenIndex() < 0) {
        return two;
    }
    if (two.getTokenIndex() < 0) {
        return one;
    }
    if (one.getTokenIndex() < two.getTokenIndex()) {
        return one;
    }
    return two;
}
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
/** Returns the token that comes last in the input stream. */
private Token getMax(Token one, Token two) {
    if (one.getTokenIndex() < 0) {
        return two;
    }
    if (two.getTokenIndex() < 0) {
        return one;
    }
    if (one.getTokenIndex() > two.getTokenIndex()) {
        return one;
    }
    return two;
}
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
protected void appendAllTokens() {
	for (int x = lastConsumedIndex + 1; input.size() > x; input.consume(), x++) {
		Token hidden = input.get(x);
		createLeafNode(hidden, null);
		lastConsumedIndex = hidden.getTokenIndex();
	}
	if (currentError != null) {
		appendError(getLastLeafNode());
	}
}
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
protected void appendTrailingHiddenTokens() {
	Token tokenBefore = input.LT(-1);
	int size = input.size();
	if (tokenBefore != null && tokenBefore.getTokenIndex() < size) {
		for (int x = tokenBefore.getTokenIndex() + 1; x < size; x++) {
			Token hidden = input.get(x);
			createLeafNode(hidden, null);
			lastConsumedIndex = hidden.getTokenIndex();
		}
	}
}
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
protected void newLeafNode(Token token, EObject grammarElement) {
	if (token == null)
		return;

	final int tokenIndex = token.getTokenIndex();
	if (tokenIndex > lastConsumedIndex) {
		for (int x = lastConsumedIndex + 1; x < tokenIndex; x++) {
			Token hidden = input.get(x);
			createLeafNode(hidden, null);
		}
		lastConsumedIndex = tokenIndex;
		lastConsumedNode = createLeafNode(token, grammarElement);
	}
}