Java TokenStream.get方法代碼示例

import org.antlr.runtime.TokenStream; //導入方法依賴的package包/類
/**
 * Returns true if there was an unexpected EOL.
 */
public static boolean hasDisallowedEOL(Callback callback) {
	TokenStream input = callback.getInput();
	Token lt = input.LT(1);

	// Start on the position before the current token and scan backwards off channel tokens until the previous on
	// channel token.
	for (int ix = lt.getTokenIndex() - 1; ix > 0; ix--) {
		lt = input.get(ix);
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
			// On channel token found: stop scanning.
			break;
		} else if (isSemicolonEquivalent(lt)) {
			return true;
		}
	}
	return false;
}
 

import org.antlr.runtime.TokenStream; //導入方法依賴的package包/類
private String extractLine(TokenStream input, RecognitionException e) {
	int line = e.line;
	// select tokens on the same line
	boolean done =false;
	int position = -1;
	StringBuilder buf = new StringBuilder();
	while(!done) {
		position++;
		try {
			Token tok = input.get(position);
			if (tok.getLine() < line)
				continue;
			if (tok.getLine() == line) {
				buf.append(tok.getText());
				buf.append(" ");
			} else if (tok.getLine() > line)
				done = true;
		} catch (NoSuchElementException nsee) {
			done = true;
		}
	}    	
	return buf.toString();	
}
 

import org.antlr.runtime.TokenStream; //導入方法依賴的package包/類
/**
 * Appends a query snippet to the message to help the user to understand the problem.
 *
 * @param parser the parser used to parse the query
 * @param builder the <code>StringBuilder</code> used to build the error message
 */
private void appendQuerySnippet(Parser parser, StringBuilder builder)
{
    TokenStream tokenStream = parser.getTokenStream();
    int index = tokenStream.index();
    int size = tokenStream.size();

    Token from = tokenStream.get(getSnippetFirstTokenIndex(index));
    Token to = tokenStream.get(getSnippetLastTokenIndex(index, size));
    Token offending = tokenStream.get(getOffendingTokenIndex(index, size));

    appendSnippet(builder, from, to, offending);
}
 

import org.antlr.runtime.TokenStream; //導入方法依賴的package包/類
/**
 * <p>
 * Promotes EOL which may lead to an automatically inserted semicolon. This is probably the most important method
 * for automatic semicolon insertion, as it is only possible to insert a semicolon in case of line breaks (even if
 * they are hidden in a multi-line comment!).
 * </p>
 */
public static void promoteEOL(Callback callback) {
	RecognizerSharedState state = callback.getState();
	TokenStream input = callback.getInput();
	// Don't promote EOL if there was a syntax error at EOF
	if (state.lastErrorIndex == input.size()) {
		return;
	}
	// Get current token and its type (the possibly offending token).
	Token prev = input.LT(-1);
	Token next = input.LT(1);
	int la = next.getType();

	// Promoting an EOL means switching it from off channel to on channel.
	// A ML_COMMENT gets promoted when it contains an EOL.
	for (int idx = prev == null ? 0 : prev.getTokenIndex() + 1, max = la == Token.EOF ? input.size()
			: next.getTokenIndex(); idx < max; idx++) {
		Token lt = input.get(idx);
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
			// On channel token found: stop scanning (previously promoted)
			break;
		} else if (isSemicolonEquivalent(lt)) {
			// We found our EOL: promote the token to on channel, position the input on it and reset the rule
			// start.
			lt.setChannel(Token.DEFAULT_CHANNEL);
			input.seek(idx);
			break;
		}
	}
}
 

import org.antlr.runtime.TokenStream; //導入方法依賴的package包/類
/**
 * A "," cannot be followed by an automatically inserted semicolon. This is in particular true in case of variable
 * statements, in which the last declaration is ended with a comma (which might easily happen in case of copying the
 * initializer from a list or object literal (cf. IDEBUG-214).
 */
private static boolean findCommaBeforeEOL(TokenStream casted, int startIndex) {
	for (int ix = startIndex - 1; ix > 0; ix--) {
		Token lt = casted.get(ix);
		if (lt.getType() == InternalN4JSParser.Comma) {
			// System.out.println("Found Comma, EOL is not valid");
			return true;
		}
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) { // any other real char ends this search
			break;
		}
	}
	return false;
}
 

import org.antlr.runtime.TokenStream; //導入方法依賴的package包/類
@Override
public String getText(TokenStream tns) {
	CommonToken leftToken = (CommonToken) tns.get(tree.getTokenStartIndex());
	CommonToken rightToken = (CommonToken) tns.get(tree.getTokenStopIndex());
	String otxt = leftToken.getInputStream().substring(leftToken.getStartIndex(), rightToken.getStopIndex());
	return otxt;
}
 

import org.antlr.runtime.TokenStream; //導入方法依賴的package包/類
/**
 * Recover from an error found on the input stream. This is for {@link NoViableAltException} and
 * {@link MismatchedTokenException}. If you enable single token insertion and deletion, this will usually not handle
 * mismatched symbol exceptions but there could be a mismatched token that the
 * {@link Parser#match(IntStream, int, BitSet) match} routine could not recover from.
 */
public static void recover(IntStream inputStream, RecognitionException re, Callback callback) {
	RecognizerSharedState state = callback.getState();
	if (re instanceof MismatchedTokenException) {
		// We expected a specific token
		// if that is not a semicolon, ASI is pointless, perform normal recovery
		int expecting = ((MismatchedTokenException) re).expecting;
		if (expecting != InternalN4JSParser.Semicolon) {

			callback.discardError(); // delete ASI message, a previously added ASI may fix too much! cf.
			// IDEBUG-215
			callback.recoverBase(inputStream, re);
			return;
		}
	}

	// System.out.println("Line: " + re.line + ":" + re.index);

	int unexpectedTokenType = re.token.getType();
	if (!followedBySemicolon(state, callback.getRecoverySets(), re.index)
			|| isOffendingToken(unexpectedTokenType)) {
		callback.recoverBase(inputStream, re);
	} else {
		int la = inputStream.LA(1);
		TokenStream casted = (TokenStream) inputStream;
		if (!isOffendingToken(la)) {
			// Start on the position before the current token and scan backwards off channel tokens until the
			// previous on channel token.
			for (int ix = re.token.getTokenIndex() - 1; ix > 0; ix--) {
				Token lt = casted.get(ix);
				if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
					// On channel token found: stop scanning.
					callback.recoverBase(inputStream, re);
					return;
				} else if (lt.getType() == InternalN4JSParser.RULE_EOL) {
					// We found our EOL: everything's good, no need to do additional recovering
					// rule start.
					if (!callback.allowASI(re)) {
						callback.recoverBase(inputStream, re);
						return;
					}
					if (!findCommaBeforeEOL(casted, ix)) {
						callback.addASIMessage();
						return;
					}
				} else if (lt.getType() == InternalN4JSParser.RULE_ML_COMMENT) {
					String tokenText = lt.getText();
					if (!findCommaBeforeEOL(casted, ix)
							&& (tokenText.indexOf('\n', 2) >= 2 || tokenText.indexOf('\r', 2) >= 2)) {
						callback.addASIMessage();
						return;
					}
				}
			}
			callback.recoverBase(inputStream, re);
		}
	}
}