Java Token.getType方法代码示例

import org.antlr.runtime.Token; //导入方法依赖的package包/类
/**
 * Returns true if the given token is considered to be semicolon equivalent.
 */
static boolean isSemicolonEquivalent(Token lt) {
	if (lt.getType() == InternalN4JSParser.RULE_EOL) {
		return true;
	}
	if (lt.getType() == InternalN4JSParser.RULE_ML_COMMENT) {
		String tokenText = lt.getText();
		for (int i = 2; i < tokenText.length() - 2; i++) {
			char c = tokenText.charAt(i);
			if (c == '\n' || c == '\r') {
				return true;
			}
		}
	}
	return false;
}
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
/** Consume tokens until one matches the given token set */
    @Override
    public void consumeUntil(IntStream i, BitSet set) {
//        System.out.println("consumeUntil(" + set.toString(getTokenNames()) + ")");
        Token ttype;
        List<Token> skipped = new ArrayList<>();
        beginResync();
        try {
            while ((ttype = input.LT(1)) != null && ttype.getType() != Token.EOF && !set.member(ttype.getType())) {
//            System.out.println("consume during recover LA(1)=" + getTokenNames()[input.LA(1)]);
                input.consume();
                skipped.add(ttype);
            }
        } finally {
            endResync();
        }
        ((NbParseTreeBuilder) dbg).consumeSkippedTokens(skipped);
    }
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
/** Creates a qualified name tree by extending an existing one
 * with an additional fragment in front.
 * @param init token holding the additional level text
 * @param subTree the existing tree; may be {@code null}, in which case
 * the result is calculated using {@link #toQualName(Token,Token)} with first
 * argument {@code null}
 */
CommonTree toQualName(Token init, CtrlTree subTree) {
    CtrlTree result;
    if (subTree == null || this.namespace.hasErrors()) {
        result = toQualName(null, init);
    } else {
        Token subTop = subTree.getToken();
        QualName qualName = subTree.getQualName()
            .nest(getText(init));
        CommonToken top = new CommonToken(subTop.getType(), qualName.toString());
        top.setLine(init.getLine());
        top.setTokenIndex(init.getTokenIndex());
        result = new CtrlTree(top);
        result.setQualName(qualName);
        result.addChild(subTree.getChild(0));
    }
    return result;
}
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
/** Create singleton template for use with dictionary values. */

    public ST createSingleton(Token templateToken) {
        String template;
        if ( templateToken.getType()==GroupParser.BIGSTRING || templateToken.getType()==GroupParser.BIGSTRING_NO_NL ) {
            template = Misc.strip(templateToken.getText(), 2);
        }
        else {
            template = Misc.strip(templateToken.getText(), 1);
        }

        CompiledST impl = compile(getFileName(), null, null, template, templateToken);
        ST st = createStringTemplateInternally(impl);
        st.groupThatCreatedThisInstance = this;
        st.impl.hasFormalArgs = false;
        st.impl.name = ST.UNKNOWN_NAME;
        st.impl.defineImplicitlyDefinedTemplates(this);
        return st;
    }
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
/** Create singleton template for use with dictionary values. */

    public ST createSingleton(Token templateToken) {
        String template;
        if ( templateToken.getType()== GroupParser.BIGSTRING || templateToken.getType()== GroupParser.BIGSTRING_NO_NL ) {
            template = Misc.strip(templateToken.getText(), 2);
        }
        else {
            template = Misc.strip(templateToken.getText(), 1);
        }

        CompiledST impl = compile(getFileName(), null, null, template, templateToken);
        ST st = createStringTemplateInternally(impl);
        st.groupThatCreatedThisInstance = this;
        st.impl.hasFormalArgs = false;
        st.impl.name = ST.UNKNOWN_NAME;
        st.impl.defineImplicitlyDefinedTemplates(this);
        return st;
    }
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
/** Create singleton template for use with dictionary values. */

    public ST createSingleton(Token templateToken) {
        String template;
        if ( templateToken.getType()== GroupParser.BIGSTRING ||
             templateToken.getType()== GroupParser.BIGSTRING_NO_NL ) {
            template = Misc.strip(templateToken.getText(), 2);
        }
        else {
            template = Misc.strip(templateToken.getText(), 1);
        }

        CompiledST impl = compile(getFileName(), null, null, template, templateToken);
        ST st = createStringTemplateInternally(impl);
        st.groupThatCreatedThisInstance = this;
        st.impl.hasFormalArgs = false;
        st.impl.name = ST.UNKNOWN_NAME;
        st.impl.defineImplicitlyDefinedTemplates(this);
        return st;
    }
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
/**
 * Produce an Antlr token for the prefix of the given leaf that overlaps the requested region
 *
 * @see #endOffset
 */
private Token toPrefixToken(ILeafNode leaf) {
	Lexer lexer = new InternalN4JSLexer();
	String text = leaf.getText();
	String prefix = text.substring(0, endOffset - leaf.getTotalOffset());
	ANTLRStringStream stream = new ANTLRStringStream(prefix);
	lexer.setCharStream(stream);
	Token nextToken = lexer.nextToken();
	// copy to get rid of the reference to the stream again
	return new CommonToken(nextToken.getType(), nextToken.getText());
}
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
/**
 * Adds token and adjust channel.
 */
@Override
public boolean add(Token tok) {
	super.add(tok);
	int type = tok.getType();
	if (type == InternalN4JSParser.EqualsSignGreaterThanSign) {
		// The arrow expression may not follow a semicolon thus we promote those here
		// to he default channel if they precede the arrow => operator
		for (int i = size() - 2; i >= 0; i--) {
			Token prev = get(i);
			if (prev.getChannel() == Token.HIDDEN_CHANNEL) {
				if (SemicolonInjectionHelper.isSemicolonEquivalent(prev)) {
					prev.setChannel(Token.DEFAULT_CHANNEL);
					break;
				}
			} else {
				break;
			}
		}
	} else if (type == InternalN4JSParser.RULE_EOL
			|| type == InternalN4JSParser.RULE_ML_COMMENT
			|| type == InternalN4JSParser.RULE_WS
			|| type == InternalN4JSParser.RULE_SL_COMMENT) {
		tok.setChannel(Token.HIDDEN_CHANNEL);
	} else {
		tok.setChannel(Token.DEFAULT_CHANNEL);
	}
	return true;
}
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
/**
 * <p>
 * Promotes EOL which may lead to an automatically inserted semicolon. This is probably the most important method
 * for automatic semicolon insertion, as it is only possible to insert a semicolon in case of line breaks (even if
 * they are hidden in a multi-line comment!).
 * </p>
 */
public static void promoteEOL(Callback callback) {
	RecognizerSharedState state = callback.getState();
	TokenStream input = callback.getInput();
	// Don't promote EOL if there was a syntax error at EOF
	if (state.lastErrorIndex == input.size()) {
		return;
	}
	// Get current token and its type (the possibly offending token).
	Token prev = input.LT(-1);
	Token next = input.LT(1);
	int la = next.getType();

	// Promoting an EOL means switching it from off channel to on channel.
	// A ML_COMMENT gets promoted when it contains an EOL.
	for (int idx = prev == null ? 0 : prev.getTokenIndex() + 1, max = la == Token.EOF ? input.size()
			: next.getTokenIndex(); idx < max; idx++) {
		Token lt = input.get(idx);
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
			// On channel token found: stop scanning (previously promoted)
			break;
		} else if (isSemicolonEquivalent(lt)) {
			// We found our EOL: promote the token to on channel, position the input on it and reset the rule
			// start.
			lt.setChannel(Token.DEFAULT_CHANNEL);
			input.seek(idx);
			break;
		}
	}
}
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
/**
 * A "," cannot be followed by an automatically inserted semicolon. This is in particular true in case of variable
 * statements, in which the last declaration is ended with a comma (which might easily happen in case of copying the
 * initializer from a list or object literal (cf. IDEBUG-214).
 */
private static boolean findCommaBeforeEOL(TokenStream casted, int startIndex) {
	for (int ix = startIndex - 1; ix > 0; ix--) {
		Token lt = casted.get(ix);
		if (lt.getType() == InternalN4JSParser.Comma) {
			// System.out.println("Found Comma, EOL is not valid");
			return true;
		}
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) { // any other real char ends this search
			break;
		}
	}
	return false;
}
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
@Override
public void consumeToken(Token token) {
    if (backtracking > 0 || resync) {
        return;
    }

    if (debug_tokens) {
        CommonToken ct = (CommonToken) token;
        int[] ctr = CommonTokenUtil.getCommonTokenOffsetRange(ct);
        System.out.println(token + "(" + ctr[0] + "-" + ctr[1] + ")");
    }

    //ignore the closing EOF token, we do not want it
    //it the parse tree
    if (token.getType() == Css3Lexer.EOF) {
        return;
    }

    //also ignore error tokens - they are added as children of ErrorNode-s in the recognitionException(...) method
    if (token.getType() == Token.INVALID_TOKEN_TYPE) {
        return;
    }

    lastConsumedToken = (CommonToken) token;

    RuleNode ruleNode = callStack.peek();
    TokenNode elementNode = new TokenNode(source, (CommonToken) token);
    elementNode.hiddenTokens = this.hiddenTokens;
    hiddenTokens.clear();
    ruleNode.addChild(elementNode);

    updateFirstTokens(ruleNode, lastConsumedToken);
}
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
protected boolean isSameTokenSequence(TokenSource originalSource, TokenSource newSource, int expectedLength) {
	Token token = originalSource.nextToken();
	int newLength = 0;
	while(Token.EOF != token.getType()) {
		Token newToken = newSource.nextToken();
		if (token.getType() != newToken.getType()) {
			return false;
		}
		newLength += TokenTool.getLength(newToken);
		token = originalSource.nextToken();
	}
	return newLength == expectedLength;
}
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
@Override
public int LA(int i) {
	Token lookaheadToken = LT(i);
	int result = lookaheadToken.getType();
	if (result == Token.EOF && getListener() != null)
		getListener().announceEof(i);
	return result;
}
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
private ILeafNode createLeafNode(Token token, EObject grammarElement) {
	boolean isHidden = token.getChannel() == HIDDEN;
	SyntaxErrorMessage error = null;
	if (!isHidden) {
		if (currentError != null) {
			error = currentError;
			currentError = null;
		}
	}
	if (token.getType() == Token.INVALID_TOKEN_TYPE) {
		if (error == null) {
			String lexerErrorMessage = ((XtextTokenStream) input).getLexerErrorMessage(token);
			LexerErrorContext errorContext = new LexerErrorContext(lexerErrorMessage);
			error = syntaxErrorProvider.getSyntaxErrorMessage(errorContext);
		}
	}
	if (grammarElement == null) {
		String ruleName = antlrTypeToLexerName.get(token.getType());
		grammarElement = allRules.get(ruleName);
	}
	CommonToken commonToken = (CommonToken) token;
	if (error != null)
		hadErrors = true;
	return nodeBuilder.newLeafNode(
			commonToken.getStartIndex(), 
			commonToken.getStopIndex() - commonToken.getStartIndex() + 1, 
			grammarElement, 
			isHidden, 
			error, 
			currentNode);
}
 

import org.antlr.runtime.Token; //导入方法依赖的package包/类
/**
 * Recover from an error found on the input stream. This is for {@link NoViableAltException} and
 * {@link MismatchedTokenException}. If you enable single token insertion and deletion, this will usually not handle
 * mismatched symbol exceptions but there could be a mismatched token that the
 * {@link Parser#match(IntStream, int, BitSet) match} routine could not recover from.
 */
public static void recover(IntStream inputStream, RecognitionException re, Callback callback) {
	RecognizerSharedState state = callback.getState();
	if (re instanceof MismatchedTokenException) {
		// We expected a specific token
		// if that is not a semicolon, ASI is pointless, perform normal recovery
		int expecting = ((MismatchedTokenException) re).expecting;
		if (expecting != InternalN4JSParser.Semicolon) {

			callback.discardError(); // delete ASI message, a previously added ASI may fix too much! cf.
			// IDEBUG-215
			callback.recoverBase(inputStream, re);
			return;
		}
	}

	// System.out.println("Line: " + re.line + ":" + re.index);

	int unexpectedTokenType = re.token.getType();
	if (!followedBySemicolon(state, callback.getRecoverySets(), re.index)
			|| isOffendingToken(unexpectedTokenType)) {
		callback.recoverBase(inputStream, re);
	} else {
		int la = inputStream.LA(1);
		TokenStream casted = (TokenStream) inputStream;
		if (!isOffendingToken(la)) {
			// Start on the position before the current token and scan backwards off channel tokens until the
			// previous on channel token.
			for (int ix = re.token.getTokenIndex() - 1; ix > 0; ix--) {
				Token lt = casted.get(ix);
				if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
					// On channel token found: stop scanning.
					callback.recoverBase(inputStream, re);
					return;
				} else if (lt.getType() == InternalN4JSParser.RULE_EOL) {
					// We found our EOL: everything's good, no need to do additional recovering
					// rule start.
					if (!callback.allowASI(re)) {
						callback.recoverBase(inputStream, re);
						return;
					}
					if (!findCommaBeforeEOL(casted, ix)) {
						callback.addASIMessage();
						return;
					}
				} else if (lt.getType() == InternalN4JSParser.RULE_ML_COMMENT) {
					String tokenText = lt.getText();
					if (!findCommaBeforeEOL(casted, ix)
							&& (tokenText.indexOf('\n', 2) >= 2 || tokenText.indexOf('\r', 2) >= 2)) {
						callback.addASIMessage();
						return;
					}
				}
			}
			callback.recoverBase(inputStream, re);
		}
	}
}