Java CommonToken.getType方法代碼示例

import org.antlr.runtime.CommonToken; //導入方法依賴的package包/類
/**
 * Returns true if the ASI at the end should be skipped for a second pass of the CA parser. Otherwise false.
 */
private boolean shouldSkipASI(CommonToken lastToken) {
	if (lastToken.getType() == eol) {
		return true;
	}
	if (lastToken.getType() == semi && lastToken.getText() != null && !";".equals(lastToken.getText())) {
		return true;
	}
	return false;
}
 

import org.antlr.runtime.CommonToken; //導入方法依賴的package包/類
/**
 * JSDoc comments are identified by the lexer as normal ML comments to simplify the ASI code. If a comment starts
 * with the sequence @çode{/**} it is remapped to a JS Doc token.
 */
@Override
protected TokenInfo createTokenInfo(CommonToken token) {
	if (token.getType() == InternalN4JSParser.RULE_ML_COMMENT) {
		String text = token.getText();
		if (text.length() > 4 && text.startsWith("/**") && text.charAt(3) != '*') {
			CommonToken jsDoc = new CommonToken(token);
			jsDoc.setType(JS_DOC_TOKEN);
			return super.createTokenInfo(jsDoc);
		}
	}
	return super.createTokenInfo(token);
}
 

import org.antlr.runtime.CommonToken; //導入方法依賴的package包/類
public static void dumpTokens(Css3Lexer lexer) {
    System.out.println("Tokens:");
    CommonToken t;
    while ((t = (CommonToken)lexer.nextToken()) != null) {
        System.out.println(
                t.getStartIndex() + "-" + t.getStopIndex() 
                + ": " + t.getText() + "(" + (t.getType() == -1 ? "" : Css3Parser.tokenNames[t.getType()]) + ")");
        
        if(t.getType() == Css3Lexer.EOF) {
            break;
        }
    }
    System.out.println("-------------");
}
 

import org.antlr.runtime.CommonToken; //導入方法依賴的package包/類
/**
 * see BUG 234135: Comments on EOF not detected
 * https://bugs.eclipse.org/bugs/show_bug.cgi?id=234135
 */
@Test public void testSlCommentEOF() {
	String model = "a\n//sl comment";
	InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
	lexer.setCharStream(new ANTLRStringStream(model));
	CommonTokenStream stream = new CommonTokenStream(lexer);
	Object eofLineComment = stream.getTokens().get(2);
	assertTrue(eofLineComment instanceof CommonToken);
	CommonToken commonToken = (CommonToken) eofLineComment;
	int type = commonToken.getType();
	assertEquals(InternalSimpleExpressionsTestLanguageLexer.RULE_SL_COMMENT, type);
}
 

import org.antlr.runtime.CommonToken; //導入方法依賴的package包/類
@Override
protected Expression instantiate(Object oldInstance, Encoder out) {
  CommonToken ct = (CommonToken)oldInstance;
  Object[] args = {ct.getType(), ct.getText()};
  return new Expression(ct, ct.getClass(), "new", args);
}
 

import org.antlr.runtime.CommonToken; //導入方法依賴的package包/類
/**
 * Returns a pointer to the start and end of the token image in the
 * underlaying stream. The token.getStopIndex() points to the last character
 * of the token which is a bit confusing.
 *
 * Use this method to get CommonToken's boundaries instead of using the
 * getStart/StopIndex methods.
 *
 * @return two members array - arr[0] is the start offset, arr[1] is the end
 * offset
 */
public static int[] getCommonTokenOffsetRange(CommonToken token) {
    if (token.getType() == CommonToken.EOF) {
        //"eof token" points at the end offset of the source, with zero length
        return new int[]{token.getStartIndex(), token.getStopIndex()};
    } else {
        return new int[]{token.getStartIndex(), token.getStopIndex() + 1};
    }

}