Java Token类代码示例

import org.antlr.runtime.Token; //导入依赖的package包/类
/**
 * Parses the input and returns a list of lexer tokens. Asserts that the produced tokens are equal to the tokens
 * that the production parser produced.
 *
 * @return the tokens for the highlighting.
 */
public List<Token> getTokens(CharSequence input) {
	List<Token> result;
	IParseResult parseResult = parser.parse(new StringReader(input.toString()));
	if (!parseResult.hasSyntaxErrors()) {
		result = throwingHighlightingParser.getTokens(input);
	} else {
		result = highlightingParser.getTokens(input);
	}
	// assert equal tokens
	Iterator<Token> iter = result.iterator();
	for (ILeafNode leaf : parseResult.getRootNode().getLeafNodes()) {
		Assert.assertTrue("hasNext at index " + leaf.getTotalOffset() + " for leaf '" + leaf.getText() + "'",
				iter.hasNext());
		Token token = iter.next();
		// TODO: assert token type
		Assert.assertEquals(leaf.getText(), token.getText());
	}
	return result;
}
 

import org.antlr.runtime.Token; //导入依赖的package包/类
@Override
protected void createContextsForLastCompleteNode(EObject previousModel, boolean strict) {
	String currentNodePrefix = getPrefix(currentNode);
	if (!Strings.isEmpty(currentNodePrefix) && !currentNode.getText().equals(currentNodePrefix)) {
		lexer.setCharStream(new ANTLRStringStream(currentNodePrefix));
		Token token = lexer.nextToken();
		if (token == Token.EOF_TOKEN) { // error case - nothing could be parsed
			return;
		}
		while (token != Token.EOF_TOKEN) {
			if (isErrorToken(token))
				return;
			token = lexer.nextToken();
		}
	}
	String prefix = "";
	Collection<FollowElement> followElements = getParser().getFollowElements(rootNode, 0, completionOffset, strict);
	doCreateContexts(lastCompleteNode, currentNode, prefix, previousModel, followElements);
}
 

import org.antlr.runtime.Token; //导入依赖的package包/类
@Override
protected TokenSource createTokenSource(String string) {
	List<Token> tokens = highlightingParser.getTokens(string);
	Iterator<Token> iter = tokens.iterator();
	return new TokenSource() {

		@Override
		public Token nextToken() {
			if (iter.hasNext()) {
				return iter.next();
			}
			return Token.EOF_TOKEN;
		}

		@Override
		public String getSourceName() {
			return "Text: " + string;
		}
	};
}
 

import org.antlr.runtime.Token; //导入依赖的package包/类
@Override
protected void announce(Token start, Token stop, AbstractElement element) {
	if (start != null && start != Token.EOF_TOKEN) {
		if (start == stop) {
			announce(start, element);
		} else {
			CommonToken castedStart = (CommonToken) start;
			if (stop == null) { // possible error condition
				if (start.getTokenIndex() == state.lastErrorIndex) {
					return;
				}
			}
			CommonToken castedEnd = (CommonToken) stop;
			Integer newType = rewriter.rewrite(castedStart, element);
			if (newType != null && castedEnd != null && castedEnd != Token.EOF_TOKEN) {
				LazyTokenStream castedInput = (LazyTokenStream) this.input;
				for (int i = castedStart.getTokenIndex() + 1; i < castedEnd.getTokenIndex(); i++) {
					Token token = castedInput.get(i);
					if (token.getChannel() != Token.HIDDEN_CHANNEL)
						token.setType(newType);
				}
				castedEnd.setType(newType);
			}
		}
	}
}
 

import org.antlr.runtime.Token; //导入依赖的package包/类
/**
 * Returns {@code true} if the set of expected follow-states includes an implicit or explicit semicolon.
 */
private static boolean followedBySemicolon(RecognizerSharedState state, Callback.RecoverySets recoverySets,
		int currentIndex) {
	int top = state._fsp;
	if (currentIndex != state.lastErrorIndex) {
		long[] array = state.following[top].toPackedArray();
		if (array.length == 1 && array[0] == (1L << Token.EOR_TOKEN_TYPE)) {
			return false;
		}
	}
	for (int i = top; i >= 0; i--) {
		BitSet localFollowSet = state.following[i];
		if (recoverySets.matches(localFollowSet)) {
			return true;
		}

	}
	return false;
}
 

import org.antlr.runtime.Token; //导入依赖的package包/类
/**
 * Returns true if there was an unexpected EOL.
 */
public static boolean hasDisallowedEOL(Callback callback) {
	TokenStream input = callback.getInput();
	Token lt = input.LT(1);

	// Start on the position before the current token and scan backwards off channel tokens until the previous on
	// channel token.
	for (int ix = lt.getTokenIndex() - 1; ix > 0; ix--) {
		lt = input.get(ix);
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
			// On channel token found: stop scanning.
			break;
		} else if (isSemicolonEquivalent(lt)) {
			return true;
		}
	}
	return false;
}
 

import org.antlr.runtime.Token; //导入依赖的package包/类
/**
 * Returns true if the given token is considered to be semicolon equivalent.
 */
static boolean isSemicolonEquivalent(Token lt) {
	if (lt.getType() == InternalN4JSParser.RULE_EOL) {
		return true;
	}
	if (lt.getType() == InternalN4JSParser.RULE_ML_COMMENT) {
		String tokenText = lt.getText();
		for (int i = 2; i < tokenText.length() - 2; i++) {
			char c = tokenText.charAt(i);
			if (c == '\n' || c == '\r') {
				return true;
			}
		}
	}
	return false;
}
 

import org.antlr.runtime.Token; //导入依赖的package包/类
/** Consume tokens until one matches the given token set */
    @Override
    public void consumeUntil(IntStream i, BitSet set) {
//        System.out.println("consumeUntil(" + set.toString(getTokenNames()) + ")");
        Token ttype;
        List<Token> skipped = new ArrayList<>();
        beginResync();
        try {
            while ((ttype = input.LT(1)) != null && ttype.getType() != Token.EOF && !set.member(ttype.getType())) {
//            System.out.println("consume during recover LA(1)=" + getTokenNames()[input.LA(1)]);
                input.consume();
                skipped.add(ttype);
            }
        } finally {
            endResync();
        }
        ((NbParseTreeBuilder) dbg).consumeSkippedTokens(skipped);
    }
 

import org.antlr.runtime.Token; //导入依赖的package包/类
/** Creates a qualified name tree by extending an existing one
 * with an additional fragment in front.
 * @param init token holding the additional level text
 * @param subTree the existing tree; may be {@code null}, in which case
 * the result is calculated using {@link #toQualName(Token,Token)} with first
 * argument {@code null}
 */
CommonTree toQualName(Token init, CtrlTree subTree) {
    CtrlTree result;
    if (subTree == null || this.namespace.hasErrors()) {
        result = toQualName(null, init);
    } else {
        Token subTop = subTree.getToken();
        QualName qualName = subTree.getQualName()
            .nest(getText(init));
        CommonToken top = new CommonToken(subTop.getType(), qualName.toString());
        top.setLine(init.getLine());
        top.setTokenIndex(init.getTokenIndex());
        result = new CtrlTree(top);
        result.setQualName(qualName);
        result.addChild(subTree.getChild(0));
    }
    return result;
}
 

import org.antlr.runtime.Token; //导入依赖的package包/类
public CompiledST defineRegion(String enclosingTemplateName, Token regionT, String template, Token templateToken) {
    String name = regionT.getText();
    template = Misc.trimOneStartingNewline(template);
    template = Misc.trimOneTrailingNewline(template);
    CompiledST code = compile(getFileName(), enclosingTemplateName, null, template, templateToken);
    String mangled = getMangledRegionName(enclosingTemplateName, name);
    if ( lookupTemplate(mangled)==null ) {
        errMgr.compileTimeError(ErrorType.NO_SUCH_REGION, templateToken, regionT, enclosingTemplateName, name);
        return new CompiledST();
    }
    code.name = mangled;
    code.isRegion = true;
    code.regionDefType = ST.RegionType.EXPLICIT;
    code.templateDefStartToken = regionT;
    rawDefineTemplate(mangled, code, regionT);
    code.defineArgDefaultValueTemplates(this);
    code.defineImplicitlyDefinedTemplates(this);
    return code;
}
 

import org.antlr.runtime.Token; //导入依赖的package包/类
public CompiledST defineTemplate(String fullyQualifiedTemplateName,
                                 Token nameT,
                                 List<FormalArgument> args,
                                 String template, Token templateToken) {
    if ( verbose ) System.out.println("defineTemplate("+fullyQualifiedTemplateName+")");
    if ( fullyQualifiedTemplateName==null || fullyQualifiedTemplateName.length()==0 ) {
        throw new IllegalArgumentException("empty template name");
    }
    if ( fullyQualifiedTemplateName.indexOf('.')>=0 ) {
        throw new IllegalArgumentException("cannot have '.' in template names");
    }
    template = Misc.trimOneStartingNewline(template);
    template = Misc.trimOneTrailingNewline(template);
    // compile, passing in templateName as enclosing name for any embedded regions
    CompiledST code = compile(getFileName(),
                              fullyQualifiedTemplateName,
                              args,
                              template,
                              templateToken);
    code.name = fullyQualifiedTemplateName;
    rawDefineTemplate(fullyQualifiedTemplateName, code, nameT);
    code.defineArgDefaultValueTemplates(this);
    code.defineImplicitlyDefinedTemplates(this); // define any anonymous subtemplates
    return code;
}
 

import org.antlr.runtime.Token; //导入依赖的package包/类
@Override
protected void createContextsForLastCompleteNode(EObject previousModel, boolean strict) {
	String currentNodePrefix = getPrefix(currentNode);
	if (!Strings.isEmpty(currentNodePrefix) && !currentNode.getText().equals(currentNodePrefix)) {
		lexer.setCharStream(new ANTLRStringStream(currentNodePrefix));
		Token token = lexer.nextToken();
		if (token == Token.EOF_TOKEN) {
			return;
		}
		while (token != Token.EOF_TOKEN) {
			if (isErrorToken(token)) {
				return;
			}
			token = lexer.nextToken();
		}
	}
	String prefix = "";
	Collection<FollowElement> followElements = parseFollowElements(completionOffset, strict);
	doCreateContexts(lastCompleteNode, currentNode, prefix, previousModel, followElements);
}
 

import org.antlr.runtime.Token; //导入依赖的package包/类
Token COMMENT() {
    match('!');
    while ( !(c=='!' && input.LA(2)==delimiterStopChar) ) {
        if ( c==EOF ) {
            RecognitionException re = new MismatchedTokenException((int)'!', input);
            re.line = input.getLine();
            re.charPositionInLine = input.getCharPositionInLine();
            errMgr.lexerError(input.getSourceName(), "Nonterminated comment starting at "+startLine+":"+startCharPositionInLine+": '!"+delimiterStopChar+"' missing", templateToken, re);
            break;
        }
        consume();
    }
    consume();
    consume(); // grab !>
    return newToken(COMMENT);
}
 

import org.antlr.runtime.Token; //导入依赖的package包/类
public CompiledST defineTemplate(String fullyQualifiedTemplateName, Token nameT, List<FormalArgument> args, String template, Token templateToken) {
    if ( verbose ) System.out.println("defineTemplate("+fullyQualifiedTemplateName+")");
    if ( fullyQualifiedTemplateName==null || fullyQualifiedTemplateName.length()==0 ) {
        throw new IllegalArgumentException("empty template name");
    }
    if ( fullyQualifiedTemplateName.indexOf('.')>=0 ) {
        throw new IllegalArgumentException("cannot have '.' in template names");
    }
    template = Misc.trimOneStartingNewline(template);
    template = Misc.trimOneTrailingNewline(template);
    // compile, passing in templateName as enclosing name for any embedded regions
    CompiledST code = compile(getFileName(), fullyQualifiedTemplateName, args, template, templateToken);
    code.name = fullyQualifiedTemplateName;
    rawDefineTemplate(fullyQualifiedTemplateName, code, nameT);
    code.defineArgDefaultValueTemplates(this);
    code.defineImplicitlyDefinedTemplates(this); // define any anonymous subtemplates
    return code;
}
 

import org.antlr.runtime.Token; //导入依赖的package包/类
Token ESCAPE() {
startCharIndex = input.index();
startCharPositionInLine = input.getCharPositionInLine();
consume(); // kill \\
if ( c=='u') return UNICODE();
String text;
      switch ( c ) {
          case '\\' : LINEBREAK(); return SKIP;
	case 'n'  : text = "\n"; break;
	case 't'  : text = "\t"; break;
	case ' '  : text = " "; break;
          default :
              NoViableAltException e = new NoViableAltException("",0,0,input);
              errMgr.lexerError(input.getSourceName(), "invalid escaped char: '"+str(c)+"'", templateToken, e);
		consume();
		match(delimiterStopChar);
		return SKIP;
      }
      consume();
Token t = newToken(TEXT, text, input.getCharPositionInLine()-2);
      match(delimiterStopChar);
      return t;
  }