Java Token.getChannel方法代碼示例

import org.antlr.runtime.Token; //導入方法依賴的package包/類
@Override
protected void announce(Token start, Token stop, AbstractElement element) {
	if (start != null && start != Token.EOF_TOKEN) {
		if (start == stop) {
			announce(start, element);
		} else {
			CommonToken castedStart = (CommonToken) start;
			if (stop == null) { // possible error condition
				if (start.getTokenIndex() == state.lastErrorIndex) {
					return;
				}
			}
			CommonToken castedEnd = (CommonToken) stop;
			Integer newType = rewriter.rewrite(castedStart, element);
			if (newType != null && castedEnd != null && castedEnd != Token.EOF_TOKEN) {
				LazyTokenStream castedInput = (LazyTokenStream) this.input;
				for (int i = castedStart.getTokenIndex() + 1; i < castedEnd.getTokenIndex(); i++) {
					Token token = castedInput.get(i);
					if (token.getChannel() != Token.HIDDEN_CHANNEL)
						token.setType(newType);
				}
				castedEnd.setType(newType);
			}
		}
	}
}
 

import org.antlr.runtime.Token; //導入方法依賴的package包/類
/**
 * Returns true if there was an unexpected EOL.
 */
public static boolean hasDisallowedEOL(Callback callback) {
	TokenStream input = callback.getInput();
	Token lt = input.LT(1);

	// Start on the position before the current token and scan backwards off channel tokens until the previous on
	// channel token.
	for (int ix = lt.getTokenIndex() - 1; ix > 0; ix--) {
		lt = input.get(ix);
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
			// On channel token found: stop scanning.
			break;
		} else if (isSemicolonEquivalent(lt)) {
			return true;
		}
	}
	return false;
}
 

import org.antlr.runtime.Token; //導入方法依賴的package包/類
/**
 * Prevent ASIs to be skipped at certain locations, e.g. after a return keyword.
 */
private boolean maySkipASI(CommonToken lastToken, ObservableXtextTokenStream tokens) {
	int countDownFrom = lastToken.getTokenIndex();
	for (int i = countDownFrom - 1; i >= 0; i--) {
		Token prevToken = tokens.get(i);
		if (prevToken.getChannel() == Token.DEFAULT_CHANNEL) {
			if (mandatoryASI.get(prevToken.getType())) {
				return false;
			}
			return true;
		}
	}
	return true;
}
 

import org.antlr.runtime.Token; //導入方法依賴的package包/類
/**
 * Adds token and adjust channel.
 */
@Override
public boolean add(Token tok) {
	super.add(tok);
	int type = tok.getType();
	if (type == InternalN4JSParser.EqualsSignGreaterThanSign) {
		// The arrow expression may not follow a semicolon thus we promote those here
		// to he default channel if they precede the arrow => operator
		for (int i = size() - 2; i >= 0; i--) {
			Token prev = get(i);
			if (prev.getChannel() == Token.HIDDEN_CHANNEL) {
				if (SemicolonInjectionHelper.isSemicolonEquivalent(prev)) {
					prev.setChannel(Token.DEFAULT_CHANNEL);
					break;
				}
			} else {
				break;
			}
		}
	} else if (type == InternalN4JSParser.RULE_EOL
			|| type == InternalN4JSParser.RULE_ML_COMMENT
			|| type == InternalN4JSParser.RULE_WS
			|| type == InternalN4JSParser.RULE_SL_COMMENT) {
		tok.setChannel(Token.HIDDEN_CHANNEL);
	} else {
		tok.setChannel(Token.DEFAULT_CHANNEL);
	}
	return true;
}
 

import org.antlr.runtime.Token; //導入方法依賴的package包/類
/**
 * <p>
 * Promotes EOL which may lead to an automatically inserted semicolon. This is probably the most important method
 * for automatic semicolon insertion, as it is only possible to insert a semicolon in case of line breaks (even if
 * they are hidden in a multi-line comment!).
 * </p>
 */
public static void promoteEOL(Callback callback) {
	RecognizerSharedState state = callback.getState();
	TokenStream input = callback.getInput();
	// Don't promote EOL if there was a syntax error at EOF
	if (state.lastErrorIndex == input.size()) {
		return;
	}
	// Get current token and its type (the possibly offending token).
	Token prev = input.LT(-1);
	Token next = input.LT(1);
	int la = next.getType();

	// Promoting an EOL means switching it from off channel to on channel.
	// A ML_COMMENT gets promoted when it contains an EOL.
	for (int idx = prev == null ? 0 : prev.getTokenIndex() + 1, max = la == Token.EOF ? input.size()
			: next.getTokenIndex(); idx < max; idx++) {
		Token lt = input.get(idx);
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
			// On channel token found: stop scanning (previously promoted)
			break;
		} else if (isSemicolonEquivalent(lt)) {
			// We found our EOL: promote the token to on channel, position the input on it and reset the rule
			// start.
			lt.setChannel(Token.DEFAULT_CHANNEL);
			input.seek(idx);
			break;
		}
	}
}
 

import org.antlr.runtime.Token; //導入方法依賴的package包/類
/**
 * A "," cannot be followed by an automatically inserted semicolon. This is in particular true in case of variable
 * statements, in which the last declaration is ended with a comma (which might easily happen in case of copying the
 * initializer from a list or object literal (cf. IDEBUG-214).
 */
private static boolean findCommaBeforeEOL(TokenStream casted, int startIndex) {
	for (int ix = startIndex - 1; ix > 0; ix--) {
		Token lt = casted.get(ix);
		if (lt.getType() == InternalN4JSParser.Comma) {
			// System.out.println("Found Comma, EOL is not valid");
			return true;
		}
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) { // any other real char ends this search
			break;
		}
	}
	return false;
}
 

import org.antlr.runtime.Token; //導入方法依賴的package包/類
private ILeafNode createLeafNode(Token token, EObject grammarElement) {
	boolean isHidden = token.getChannel() == HIDDEN;
	SyntaxErrorMessage error = null;
	if (!isHidden) {
		if (currentError != null) {
			error = currentError;
			currentError = null;
		}
	}
	if (token.getType() == Token.INVALID_TOKEN_TYPE) {
		if (error == null) {
			String lexerErrorMessage = ((XtextTokenStream) input).getLexerErrorMessage(token);
			LexerErrorContext errorContext = new LexerErrorContext(lexerErrorMessage);
			error = syntaxErrorProvider.getSyntaxErrorMessage(errorContext);
		}
	}
	if (grammarElement == null) {
		String ruleName = antlrTypeToLexerName.get(token.getType());
		grammarElement = allRules.get(ruleName);
	}
	CommonToken commonToken = (CommonToken) token;
	if (error != null)
		hadErrors = true;
	return nodeBuilder.newLeafNode(
			commonToken.getStartIndex(), 
			commonToken.getStopIndex() - commonToken.getStartIndex() + 1, 
			grammarElement, 
			isHidden, 
			error, 
			currentNode);
}
 

import org.antlr.runtime.Token; //導入方法依賴的package包/類
protected void skipHiddenTokens() {
	if (hiddenTokens == null || hiddenTokens.isEmpty())
		return;
	Token token = LT(1);
	while(token.getChannel() == Token.HIDDEN_CHANNEL) {
		p++;
		token = LT(1);
	}
}
 

import org.antlr.runtime.Token; //導入方法依賴的package包/類
/**
 * Recover from an error found on the input stream. This is for {@link NoViableAltException} and
 * {@link MismatchedTokenException}. If you enable single token insertion and deletion, this will usually not handle
 * mismatched symbol exceptions but there could be a mismatched token that the
 * {@link Parser#match(IntStream, int, BitSet) match} routine could not recover from.
 */
public static void recover(IntStream inputStream, RecognitionException re, Callback callback) {
	RecognizerSharedState state = callback.getState();
	if (re instanceof MismatchedTokenException) {
		// We expected a specific token
		// if that is not a semicolon, ASI is pointless, perform normal recovery
		int expecting = ((MismatchedTokenException) re).expecting;
		if (expecting != InternalN4JSParser.Semicolon) {

			callback.discardError(); // delete ASI message, a previously added ASI may fix too much! cf.
			// IDEBUG-215
			callback.recoverBase(inputStream, re);
			return;
		}
	}

	// System.out.println("Line: " + re.line + ":" + re.index);

	int unexpectedTokenType = re.token.getType();
	if (!followedBySemicolon(state, callback.getRecoverySets(), re.index)
			|| isOffendingToken(unexpectedTokenType)) {
		callback.recoverBase(inputStream, re);
	} else {
		int la = inputStream.LA(1);
		TokenStream casted = (TokenStream) inputStream;
		if (!isOffendingToken(la)) {
			// Start on the position before the current token and scan backwards off channel tokens until the
			// previous on channel token.
			for (int ix = re.token.getTokenIndex() - 1; ix > 0; ix--) {
				Token lt = casted.get(ix);
				if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
					// On channel token found: stop scanning.
					callback.recoverBase(inputStream, re);
					return;
				} else if (lt.getType() == InternalN4JSParser.RULE_EOL) {
					// We found our EOL: everything's good, no need to do additional recovering
					// rule start.
					if (!callback.allowASI(re)) {
						callback.recoverBase(inputStream, re);
						return;
					}
					if (!findCommaBeforeEOL(casted, ix)) {
						callback.addASIMessage();
						return;
					}
				} else if (lt.getType() == InternalN4JSParser.RULE_ML_COMMENT) {
					String tokenText = lt.getText();
					if (!findCommaBeforeEOL(casted, ix)
							&& (tokenText.indexOf('\n', 2) >= 2 || tokenText.indexOf('\r', 2) >= 2)) {
						callback.addASIMessage();
						return;
					}
				}
			}
			callback.recoverBase(inputStream, re);
		}
	}
}
 

import org.antlr.runtime.Token; //導入方法依賴的package包/類
/**
 * Fills the buffer but stops on a div or div-equals token.
 */
@SuppressWarnings("unchecked")
@Override
protected void fillBuffer() {
	int oldP = p;
	int index = tokens.size();
	Token t = tokenSource.nextToken();
	while (t != null && t.getType() != CharStream.EOF) {
		// stop on div, div-equal and right curly brace tokens tokens.
		int type = t.getType();
		if (type == InternalN4JSLexer.Solidus || type == InternalN4JSLexer.SolidusEqualsSign
				|| type == InternalN4JSLexer.RightCurlyBracket) {
			t.setTokenIndex(index);
			tokens.add(t);
			index++;
			break;
		}
		boolean discard = false;
		// is there a channel override for token type?
		if (channelOverrideMap != null) {
			Integer channelI = (Integer) channelOverrideMap.get(Integer.valueOf(type));
			if (channelI != null) {
				t.setChannel(channelI.intValue());
			}
		}
		if (discardSet != null &&
				discardSet.contains(Integer.valueOf(type))) {
			discard = true;
		} else if (discardOffChannelTokens && t.getChannel() != this.channel) {
			discard = true;
		}
		if (!discard) {
			t.setTokenIndex(index);
			tokens.add(t);
			index++;
		}
		t = tokenSource.nextToken();
	}
	// leave p pointing at first token on channel
	p = oldP == -1 ? 0 : oldP;
	p = skipOffTokenChannels(p);
}