Java Token.EOF_TOKEN属性代码示例

@Override
protected void createContextsForLastCompleteNode(EObject previousModel, boolean strict) {
	String currentNodePrefix = getPrefix(currentNode);
	if (!Strings.isEmpty(currentNodePrefix) && !currentNode.getText().equals(currentNodePrefix)) {
		lexer.setCharStream(new ANTLRStringStream(currentNodePrefix));
		Token token = lexer.nextToken();
		if (token == Token.EOF_TOKEN) { // error case - nothing could be parsed
			return;
		}
		while (token != Token.EOF_TOKEN) {
			if (isErrorToken(token))
				return;
			token = lexer.nextToken();
		}
	}
	String prefix = "";
	Collection<FollowElement> followElements = getParser().getFollowElements(rootNode, 0, completionOffset, strict);
	doCreateContexts(lastCompleteNode, currentNode, prefix, previousModel, followElements);
}
 

@Override
protected TokenSource createTokenSource(String string) {
	List<Token> tokens = highlightingParser.getTokens(string);
	Iterator<Token> iter = tokens.iterator();
	return new TokenSource() {

		@Override
		public Token nextToken() {
			if (iter.hasNext()) {
				return iter.next();
			}
			return Token.EOF_TOKEN;
		}

		@Override
		public String getSourceName() {
			return "Text: " + string;
		}
	};
}
 

@Override
protected void announce(Token start, Token stop, AbstractElement element) {
	if (start != null && start != Token.EOF_TOKEN) {
		if (start == stop) {
			announce(start, element);
		} else {
			CommonToken castedStart = (CommonToken) start;
			if (stop == null) { // possible error condition
				if (start.getTokenIndex() == state.lastErrorIndex) {
					return;
				}
			}
			CommonToken castedEnd = (CommonToken) stop;
			Integer newType = rewriter.rewrite(castedStart, element);
			if (newType != null && castedEnd != null && castedEnd != Token.EOF_TOKEN) {
				LazyTokenStream castedInput = (LazyTokenStream) this.input;
				for (int i = castedStart.getTokenIndex() + 1; i < castedEnd.getTokenIndex(); i++) {
					Token token = castedInput.get(i);
					if (token.getChannel() != Token.HIDDEN_CHANNEL)
						token.setType(newType);
				}
				castedEnd.setType(newType);
			}
		}
	}
}
 

@Override
protected void createContextsForLastCompleteNode(EObject previousModel, boolean strict) {
	String currentNodePrefix = getPrefix(currentNode);
	if (!Strings.isEmpty(currentNodePrefix) && !currentNode.getText().equals(currentNodePrefix)) {
		lexer.setCharStream(new ANTLRStringStream(currentNodePrefix));
		Token token = lexer.nextToken();
		if (token == Token.EOF_TOKEN) {
			return;
		}
		while (token != Token.EOF_TOKEN) {
			if (isErrorToken(token)) {
				return;
			}
			token = lexer.nextToken();
		}
	}
	String prefix = "";
	Collection<FollowElement> followElements = parseFollowElements(completionOffset, strict);
	doCreateContexts(lastCompleteNode, currentNode, prefix, previousModel, followElements);
}
 

protected void createContextsForLastCompleteNode(EObject previousModel, boolean strict) {
	String currentNodePrefix = getPrefix(currentNode);
	if (!Strings.isEmpty(currentNodePrefix) && !currentNode.getText().equals(currentNodePrefix)) {
		lexer.setCharStream(new ANTLRStringStream(currentNodePrefix));
		Token token = lexer.nextToken();
		if (token == Token.EOF_TOKEN) { // error case - nothing could be parsed
			return;
		}
		while(token != Token.EOF_TOKEN) {
			if (isErrorToken(token))
				return;
			token = lexer.nextToken();
		}
	}
	String prefix = "";
	String completeInput = getInputToParse(document, completionOffset);
	Collection<FollowElement> followElements = parser.getFollowElements(completeInput, strict);
	doCreateContexts(lastCompleteNode, currentNode, prefix, previousModel, followElements);
}
 

/**
 * Implementation of the {@link TokenSource} interface. Return new tokens as long as there are some, afterwards
 * return {@link Token#EOF_TOKEN}.
 */
@Override
public Token nextToken() {
	if (next != null) {
		Token result = next;
		next = null;
		return result;
	}
	if (!leafNodes.hasNext()) {
		return Token.EOF_TOKEN;
	}
	ILeafNode leaf = leafNodes.next();
	if (leaf.getTotalOffset() >= endOffset) {
		leafNodes = Iterators.emptyIterator();
		return Token.EOF_TOKEN;
	}
	if (leaf.getTotalEndOffset() <= startOffset) {
		return nextToken();
	}
	if (leaf.getTotalEndOffset() > endOffset) {
		return toPrefixToken(leaf);
	}
	SyntaxErrorMessage syntaxErrorMessage = leaf.getSyntaxErrorMessage();
	if (syntaxErrorMessage != null && SEMICOLON_INSERTED.equals(syntaxErrorMessage.getIssueCode())) {
		return toASIToken(leaf);
	}
	if (leaf.isHidden()) {
		return processHiddenToken(leaf);
	}
	int tokenType = tokenTypeMapper.getInternalTokenType(leaf);
	return new CommonToken(tokenType, leaf.getText());
}
 

/**
 * Skips the given leaf as it's hidden. If it was the last token to be returned, a hidden token may be syntesized if
 * would affect the semicolon insertion.
 */
private Token processHiddenToken(ILeafNode leaf) {
	Token result = nextToken();
	if (result == Token.EOF_TOKEN && Strings.countLineBreaks(leaf.getText()) > 0) {
		next = result;
		CommonToken hidden = new CommonToken(tokenTypeMapper.getInternalTokenType(leaf), leaf.getText());
		hidden.setChannel(Token.HIDDEN_CHANNEL);
		return hidden;
	}
	return result;
}
 

private void exhaustTokenSource() {
	LazyTokenStream casted = (LazyTokenStream) this.input;
	int marked = casted.mark();
	try {
		while (casted.LT(1) != Token.EOF_TOKEN) {
			casted.consume();
		}
	} finally {
		casted.rewind(marked);
	}
}
 

@Override
public Token nextToken() {
	while (true) {
		this.state.token = null;
		this.state.channel = Token.DEFAULT_CHANNEL;
		this.state.tokenStartCharIndex = input.index();
		this.state.tokenStartCharPositionInLine = input.getCharPositionInLine();
		this.state.tokenStartLine = input.getLine();
		this.state.text = null;
		if (input.LA(1) == CharStream.EOF) {
			return Token.EOF_TOKEN;
		}
		try {
			mTokens();
			if (this.state.token == null) {
				emit();
			}
			else if (this.state.token == Token.SKIP_TOKEN) {
				continue;
			}
			return this.state.token;
		}
		catch (RecognitionException re) {
			reportError(re);
			if (re instanceof NoViableAltException ||
				re instanceof FailedPredicateException) {
				recover(re);
			}
			// create token that holds mismatched char
			Token t = new CommonToken(input, Token.INVALID_TOKEN_TYPE, Token.HIDDEN_CHANNEL,
					this.state.tokenStartCharIndex, getCharIndex() - 1);
			t.setLine(this.state.tokenStartLine);
			t.setCharPositionInLine(this.state.tokenStartCharPositionInLine);
			tokenErrorMap.put(t, getErrorMessage(re, this.getTokenNames()));
			emit(t);
			return this.state.token;
		}
	}
}
 

@Override
public Token nextToken() {
	if (iter.hasNext()) {
		ILookAheadTerminal lookAhead = iter.next();
		return lookAhead.getToken();
	}
	return Token.EOF_TOKEN;
}
 

@Override
protected void error(NoViableAltException nvae) {
	if (nvae.token == Token.EOF_TOKEN) {
		int lookAheadAddOn = getRecognizer().lookAheadAddOn;
		int lookAhead = ((XtextTokenStream)nvae.input).getCurrentLookAhead();
		if ((lookAhead >= lookAheadAddOn && lookAheadAddOn > 0) || (lookAhead == 0 && lookAheadAddOn > 0) || lookAhead == -1)
			getRecognizer().failedPredicateAtEOF = true;
	}
	super.error(nvae);
}
 

@Override
public Token nextToken() {
	if (tokenCount == 0)
		return Token.EOF_TOKEN;
	return new CommonToken(tokenCount--, "Text");
}