Java TokenSource.nextToken方法代码示例

import org.antlr.runtime.TokenSource; //导入方法依赖的package包/类
/**
 * @since 2.4
 */
@Override
protected RepairEntryData getRepairEntryData(DocumentEvent e) throws Exception {
	int tokenStartsAt = 0;
	int tokenInfoIdx = 0;
	TokenSource source = createTokenSource(e.fDocument.get());
	CommonToken token = (CommonToken) source.nextToken();
	// find start idx
	while (true) {
		if (token == Token.EOF_TOKEN) {
			break;
		}
		if (tokenInfoIdx >= getInternalModifyableTokenInfos().size())
			break;
		TokenInfo tokenInfo = getInternalModifyableTokenInfos().get(tokenInfoIdx);
		if (tokenInfo.getAntlrTokenType() != token.getType()
				|| token.getStopIndex() - token.getStartIndex() + 1 != tokenInfo.getLength())
			break;
		if (tokenStartsAt + tokenInfo.getLength() > e.fOffset)
			break;
		tokenStartsAt += tokenInfo.getLength();
		tokenInfoIdx++;
		token = (CommonToken) source.nextToken();
	}
	return new RepairEntryData(tokenStartsAt, tokenInfoIdx, token, source);
}
 

import org.antlr.runtime.TokenSource; //导入方法依赖的package包/类
protected void assertTokens(String value, TokenSource tokenSource, String escapedString) {
	if (tokenSource == null)
		return;
	Token token = tokenSource.nextToken();
	if (!escapedString.equals(token.getText())) {
		throw createTokenContentMismatchException(value, escapedString, token);
	}
	
	String rule1 = getRuleName().toUpperCase();
	String rule2 = getRuleName(token);
	// workaround: 
	if ("NAME".equals(rule1) && "ID".equals(rule2)) {
		rule2 = "NAME";
	} else if ("ID".equals(rule1) && "NAME".equals(rule2)) {
		rule1 = "NAME";
	}
	// Ambiguity between NAME and ID rule
	if (!rule1.equals(rule2)) {
		throw createTokenTypeMismatchException(value, escapedString, token);
	}
	String reparsedValue = toValue(token.getText(), null);
	if (value != reparsedValue && !value.equals(reparsedValue)) {
		throw createTokenContentMismatchException(value, escapedString, token);
	}
}
 

import org.antlr.runtime.TokenSource; //导入方法依赖的package包/类
protected boolean isSameTokenSequence(TokenSource originalSource, TokenSource newSource, int expectedLength) {
	Token token = originalSource.nextToken();
	int newLength = 0;
	while(Token.EOF != token.getType()) {
		Token newToken = newSource.nextToken();
		if (token.getType() != newToken.getType()) {
			return false;
		}
		newLength += TokenTool.getLength(newToken);
		token = originalSource.nextToken();
	}
	return newLength == expectedLength;
}
 

import org.antlr.runtime.TokenSource; //导入方法依赖的package包/类
protected void assertTokens(T value, TokenSource tokenSource, String escapedString) {
	if (tokenSource == null)
		return;
	Token token = tokenSource.nextToken();
	if (!escapedString.equals(token.getText())) {
		throw createTokenContentMismatchException(value, escapedString, token);
	}
	if (!getRuleName().toUpperCase().equals(getRuleName(token))) {
		throw createTokenTypeMismatchException(value, escapedString, token);
	}
	T reparsedValue = toValue(token.getText(), null);
	if (value != reparsedValue && !value.equals(reparsedValue)) {
		throw createTokenContentMismatchException(value, escapedString, token);
	}
}
 

import org.antlr.runtime.TokenSource; //导入方法依赖的package包/类
protected boolean equalTokenSequence(TokenSource first, TokenSource second) {
	Token token = null;
	while (!(token = first.nextToken()).equals(Token.EOF_TOKEN)) {
		Token otherToken = second.nextToken();
		if (otherToken.equals(Token.EOF_TOKEN)) {
			return false;
		}
		if (!token.getText().equals(otherToken.getText())) {
			return false;
		}
	}
	return true;
}
 

import org.antlr.runtime.TokenSource; //导入方法依赖的package包/类
protected boolean equalTokenSequence(TokenSource first, TokenSource second) {
	Token token = null;
	while(!(token = first.nextToken()).equals(Token.EOF_TOKEN)) {
		Token otherToken = second.nextToken();
		if (otherToken.equals(Token.EOF_TOKEN)) {
			return false;
		}
		if (!token.getText().equals(otherToken.getText())) {
			return false;
		}
	}
	return true;
}
 

import org.antlr.runtime.TokenSource; //导入方法依赖的package包/类
protected List<TokenInfo> createTokenInfos(String string) {
	List<TokenInfo> result = Lists.newArrayListWithExpectedSize(string.length() / 3);
	TokenSource source = createTokenSource(string);
	CommonToken token = (CommonToken) source.nextToken();
	while (token != Token.EOF_TOKEN) {
		TokenInfo info = createTokenInfo(token);
		result.add(info);
		token = (CommonToken) source.nextToken();
	}
	return result;
}
 

import org.antlr.runtime.TokenSource; //导入方法依赖的package包/类
/**
 * @since 2.4
 */
protected RepairEntryData getRepairEntryData(DocumentEvent e) throws Exception {
	int tokenStartsAt = 0;
	int tokenInfoIdx = 0;
	for(tokenInfoIdx = 0; tokenInfoIdx< getInternalModifyableTokenInfos().size(); ++tokenInfoIdx) {
		TokenInfo oldToken = getInternalModifyableTokenInfos().get(tokenInfoIdx);
		if(tokenStartsAt <= e.getOffset() && tokenStartsAt + oldToken.getLength() >= e.getOffset())
			break;
		tokenStartsAt += oldToken.getLength();
	}
	final TokenSource delegate = createTokenSource(e.fDocument.get(tokenStartsAt, e.fDocument.getLength() - tokenStartsAt));
	final int offset = tokenStartsAt;
	TokenSource source = new TokenSource() {
		public Token nextToken() {
			CommonToken commonToken = (CommonToken) delegate.nextToken();
			commonToken.setText(commonToken.getText());
			commonToken.setStartIndex(commonToken.getStartIndex()+offset);
			commonToken.setStopIndex(commonToken.getStopIndex()+offset);
			return commonToken;
		}

		public String getSourceName() {
			return delegate.getSourceName();
		}
	};
	final CommonToken token = (CommonToken) source.nextToken();
	return new RepairEntryData(offset, tokenInfoIdx, token, source);
}
 

import org.antlr.runtime.TokenSource; //导入方法依赖的package包/类
static void printTokenStream(CommonTokenStream tokens, Map tokenToName) {
    TokenSource source = tokens.getTokenSource();
    Token curr;
    do {
        curr = source.nextToken();
        System.out.println(curr.getText() + " [" + tokenToName.get(String.valueOf(curr.getType())) + "]");
    } while (-1 != curr.getType() ); // eof
}