Java TokenSource类代码示例

import org.antlr.runtime.TokenSource; //导入依赖的package包/类
/**
 * First pass. Use the tokens that have been computed from the production parser's result and collect the follow
 * elements from those.
 */
private CustomInternalN4JSParser collectFollowElements(TokenSource tokens, boolean strict,
		Set<FollowElement> result) {
	CustomInternalN4JSParser parser = createParser();
	parser.setStrict(strict);
	try {
		ObservableXtextTokenStream tokenStream = new ObservableXtextTokenStream(tokens, parser);
		result.addAll(doGetFollowElements(parser, tokenStream));
	} catch (InfiniteRecursion infinite) {
		// this guards against erroneous infinite recovery loops in Antlr.
		// Grammar dependent and not expected something that is expected for N4JS
		// is used in the base class thus also used here.
		result.addAll(parser.getFollowElements());
	}
	return parser;
}
 

import org.antlr.runtime.TokenSource; //导入依赖的package包/类
@Override
protected TokenSource createTokenSource(String string) {
	List<Token> tokens = highlightingParser.getTokens(string);
	Iterator<Token> iter = tokens.iterator();
	return new TokenSource() {

		@Override
		public Token nextToken() {
			if (iter.hasNext()) {
				return iter.next();
			}
			return Token.EOF_TOKEN;
		}

		@Override
		public String getSourceName() {
			return "Text: " + string;
		}
	};
}
 

import org.antlr.runtime.TokenSource; //导入依赖的package包/类
protected IParseResult doParse(String ruleName, CharStream in, NodeModelBuilder nodeModelBuilder, int initialLookAhead) {
	TokenSource tokenSource = createLexer(in);
	XtextTokenStream tokenStream = createTokenStream(tokenSource);
	tokenStream.initCurrentLookAhead(initialLookAhead);
	setInitialHiddenTokens(tokenStream);
	AbstractInternalAntlrParser parser = createParser(tokenStream);
	parser.setTokenTypeMap(getTokenDefProvider().getTokenDefMap());
	parser.setSyntaxErrorProvider(getSyntaxErrorProvider());
	parser.setNodeModelBuilder(nodeModelBuilder);
	parser.setSemanticModelBuilder(getElementFactory());
	IUnorderedGroupHelper helper = getUnorderedGroupHelper().get();
	parser.setUnorderedGroupHelper(helper);
	helper.initializeWith(parser);
	try {
		if(ruleName != null)
			return parser.parse(ruleName);
		return parser.parse();
	} catch (Exception re) {
		throw new ParseException(re.getMessage(),re);
	}
}
 

import org.antlr.runtime.TokenSource; //导入依赖的package包/类
public Collection<FE> getFollowElements(String input, boolean strict) {
	TokenSource tokenSource = createTokenSource(input);
	InternalParser parser = createParser();
	parser.setStrict(strict);
	ObservableXtextTokenStream tokens = new ObservableXtextTokenStream(tokenSource, parser);
	tokens.setInitialHiddenTokens(getInitialHiddenTokens());
	parser.setTokenStream(tokens);
	IUnorderedGroupHelper helper = createUnorderedGroupHelper();
	parser.setUnorderedGroupHelper(helper);
	helper.initializeWith(parser);
	tokens.setListener(parser);
	try {
		return Lists.newArrayList(getFollowElements(parser));
	} catch (InfiniteRecursion infinite) {
		return Lists.newArrayList(parser.getFollowElements());
	}
}
 

import org.antlr.runtime.TokenSource; //导入依赖的package包/类
public Collection<FollowElement> getFollowElements(String input, boolean strict) {
	CharStream stream = new ANTLRStringStream(input);
	TokenSource tokenSource = createLexer(stream);
	AbstractInternalContentAssistParser parser = createParser();
	parser.setStrict(strict);
	ObservableXtextTokenStream tokens = new ObservableXtextTokenStream(tokenSource, parser);
	tokens.setInitialHiddenTokens(getInitialHiddenTokens());
	parser.setTokenStream(tokens);
	IUnorderedGroupHelper helper = getUnorderedGroupHelper().get();
	parser.setUnorderedGroupHelper(helper);
	helper.initializeWith(parser);
	tokens.setListener(parser);
	try {
		return Lists.newArrayList(getFollowElements(parser));
	} catch(InfiniteRecursion infinite) {
		return Lists.newArrayList(parser.getFollowElements());
	}
}
 

import org.antlr.runtime.TokenSource; //导入依赖的package包/类
/**
 * @since 2.4
 */
@Override
protected RepairEntryData getRepairEntryData(DocumentEvent e) throws Exception {
	int tokenStartsAt = 0;
	int tokenInfoIdx = 0;
	TokenSource source = createTokenSource(e.fDocument.get());
	CommonToken token = (CommonToken) source.nextToken();
	// find start idx
	while (true) {
		if (token == Token.EOF_TOKEN) {
			break;
		}
		if (tokenInfoIdx >= getInternalModifyableTokenInfos().size())
			break;
		TokenInfo tokenInfo = getInternalModifyableTokenInfos().get(tokenInfoIdx);
		if (tokenInfo.getAntlrTokenType() != token.getType()
				|| token.getStopIndex() - token.getStartIndex() + 1 != tokenInfo.getLength())
			break;
		if (tokenStartsAt + tokenInfo.getLength() > e.fOffset)
			break;
		tokenStartsAt += tokenInfo.getLength();
		tokenInfoIdx++;
		token = (CommonToken) source.nextToken();
	}
	return new RepairEntryData(tokenStartsAt, tokenInfoIdx, token, source);
}
 

import org.antlr.runtime.TokenSource; //导入依赖的package包/类
protected void assertTokens(String value, TokenSource tokenSource, String escapedString) {
	if (tokenSource == null)
		return;
	Token token = tokenSource.nextToken();
	if (!escapedString.equals(token.getText())) {
		throw createTokenContentMismatchException(value, escapedString, token);
	}
	
	String rule1 = getRuleName().toUpperCase();
	String rule2 = getRuleName(token);
	// workaround: 
	if ("NAME".equals(rule1) && "ID".equals(rule2)) {
		rule2 = "NAME";
	} else if ("ID".equals(rule1) && "NAME".equals(rule2)) {
		rule1 = "NAME";
	}
	// Ambiguity between NAME and ID rule
	if (!rule1.equals(rule2)) {
		throw createTokenTypeMismatchException(value, escapedString, token);
	}
	String reparsedValue = toValue(token.getText(), null);
	if (value != reparsedValue && !value.equals(reparsedValue)) {
		throw createTokenContentMismatchException(value, escapedString, token);
	}
}
 

import org.antlr.runtime.TokenSource; //导入依赖的package包/类
/**
 * @param node
 *            the root node of the model to parse
 * @param startOffset
 *            the start offset to consider
 * @param endOffset
 *            the exclusive end offset
 * @param strict
 *            if true the parser will not use error recovery on the very last token of the input.
 * @return a collection of follow elements.
 */
public Collection<FollowElement> getFollowElements(INode node, int startOffset, int endOffset, boolean strict) {
	Set<FollowElement> result = Sets.newLinkedHashSet();

	TokenSource tokenSource = tokenSourceFactory.toTokenSource(node, startOffset, endOffset);
	CustomInternalN4JSParser parser = collectFollowElements(tokenSource, strict, result);
	adjustASIAndCollectFollowElements(parser, strict, result);

	/*
	 * Lists are easier to debug
	 */
	return Lists.newArrayList(result);
}
 

import org.antlr.runtime.TokenSource; //导入依赖的package包/类
protected boolean isSameTokenSequence(TokenSource originalSource, TokenSource newSource, int expectedLength) {
	Token token = originalSource.nextToken();
	int newLength = 0;
	while(Token.EOF != token.getType()) {
		Token newToken = newSource.nextToken();
		if (token.getType() != newToken.getType()) {
			return false;
		}
		newLength += TokenTool.getLength(newToken);
		token = originalSource.nextToken();
	}
	return newLength == expectedLength;
}
 

import org.antlr.runtime.TokenSource; //导入依赖的package包/类
public XtextTokenStream(TokenSource tokenSource, ITokenDefProvider tokenDefProvider) {
	super(tokenSource);
	tokens = new TokenList(500);
	rulenameToTokenType = new HashMap<String, Integer>(tokenDefProvider.getTokenDefMap().size());
	for(Map.Entry<Integer, String> entry: tokenDefProvider.getTokenDefMap().entrySet()) {
		rulenameToTokenType.put(entry.getValue(), entry.getKey());
	}
}
 

import org.antlr.runtime.TokenSource; //导入依赖的package包/类
protected void assertTokens(T value, TokenSource tokenSource, String escapedString) {
	if (tokenSource == null)
		return;
	Token token = tokenSource.nextToken();
	if (!escapedString.equals(token.getText())) {
		throw createTokenContentMismatchException(value, escapedString, token);
	}
	if (!getRuleName().toUpperCase().equals(getRuleName(token))) {
		throw createTokenTypeMismatchException(value, escapedString, token);
	}
	T reparsedValue = toValue(token.getText(), null);
	if (value != reparsedValue && !value.equals(reparsedValue)) {
		throw createTokenContentMismatchException(value, escapedString, token);
	}
}
 

import org.antlr.runtime.TokenSource; //导入依赖的package包/类
protected TokenSource getTokenSource(String escapedValue) {
	Lexer result = getLexer();
	if (result == null)
		return null;
	result.setCharStream(new ANTLRStringStream(escapedValue));
	return result;
}
 

import org.antlr.runtime.TokenSource; //导入依赖的package包/类
@Override
public TokenSource createTokenSource(final CharStream stream) {
  if ((this.parser instanceof AbstractAntlrParser)) {
    return ((AbstractAntlrParser)this.parser).createLexer(stream);
  }
  StringConcatenation _builder = new StringConcatenation();
  String _name = this.parser.getClass().getName();
  _builder.append(_name);
  _builder.append(" should be a subclass of ");
  String _name_1 = AbstractAntlrParser.class.getName();
  _builder.append(_name_1);
  throw new IllegalStateException(_builder.toString());
}
 

import org.antlr.runtime.TokenSource; //导入依赖的package包/类
protected boolean equalTokenSequence(TokenSource first, TokenSource second) {
	Token token = null;
	while (!(token = first.nextToken()).equals(Token.EOF_TOKEN)) {
		Token otherToken = second.nextToken();
		if (otherToken.equals(Token.EOF_TOKEN)) {
			return false;
		}
		if (!token.getText().equals(otherToken.getText())) {
			return false;
		}
	}
	return true;
}
 

import org.antlr.runtime.TokenSource; //导入依赖的package包/类
protected boolean equalTokenSequence(TokenSource first, TokenSource second) {
	Token token = null;
	while(!(token = first.nextToken()).equals(Token.EOF_TOKEN)) {
		Token otherToken = second.nextToken();
		if (otherToken.equals(Token.EOF_TOKEN)) {
			return false;
		}
		if (!token.getText().equals(otherToken.getText())) {
			return false;
		}
	}
	return true;
}