Java Token.getTokenIndex方法代码示例

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
private List<SyntaxElement> addNonSyntaxTokens(List<SyntaxElement> syntaxElements, BufferedTokenStream tokens) {

		StackLight<SyntaxElement> regularElements = new StackLight<SyntaxElement>(syntaxElements);

		List<SyntaxElement> result = new LinkedList<SyntaxElement>();

		for (Token t : tokens.getTokens()) {
			if (!regularElements.empty() && regularElements.peek().tokenIndex == t.getTokenIndex())
				result.add(regularElements.pop());
			else {
				SyntaxElementType type = getIrregularType(t);
				if (type != SyntaxElementType.unknown)
					result.add(SyntaxElement.create(t, type));
			}
		}

		return result;
	}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
/** Walk upwards from node until we find a child of p at t's char position.
 *  Don't see alignment with self, t, or element *after* us.
 *  return null if there is no such ancestor p.
 */
public static Pair<ParserRuleContext,Integer> earliestAncestorWithChildStartingAtCharPos(ParserRuleContext node, Token t, int charpos) {
	ParserRuleContext p = node;
	while ( p!=null ) {
		// check all children of p to see if one of them starts at charpos
		for (int i = 0; i<p.getChildCount(); i++) {
			ParseTree child = p.getChild(i);
			Token start;
			if ( child instanceof ParserRuleContext ) {
				start = ((ParserRuleContext) child).getStart();
			}
			else { // must be token
				start = ((TerminalNode)child).getSymbol();
			}
			// check that we don't see alignment with self or element *after* us
			if ( start.getTokenIndex()<t.getTokenIndex() && start.getCharPositionInLine()==charpos ) {
				return new Pair<>(p,i);
			}
		}
		p = p.getParent();
	}
	return null;
}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
public static int getMatchingSymbolStartsLine(Corpus corpus,
	                                              InputDocument doc,
	                                              TerminalNode node)
	{
		TerminalNode matchingLeftNode = getMatchingLeftSymbol(corpus, doc, node);
		if ( matchingLeftNode != null ) {
			Token matchingLeftToken = matchingLeftNode.getSymbol();
			int i = matchingLeftToken.getTokenIndex();
			if ( i==0 ) return 1; // first token is considered first on line
			Token tokenBeforeMatchingToken = doc.tokens.getPreviousRealToken(i);
//			System.out.printf("doc=%s node=%s, pair=%s, before=%s\n",
//			                  new File(doc.fileName).getName(), node.getSymbol(), matchingLeftToken, tokenBeforeMatchingToken);
			if ( tokenBeforeMatchingToken!=null ) {
				return matchingLeftToken.getLine()>tokenBeforeMatchingToken.getLine() ? 1 : 0;
			}
			else { // matchingLeftToken must be first in file
				return 1;
			}
		}
		return NOT_PAIR;
	}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
public static int getMatchingSymbolEndsLine(Corpus corpus,
	                                            InputDocument doc,
	                                            TerminalNode node)
	{
		TerminalNode matchingLeftNode = getMatchingLeftSymbol(corpus, doc, node);
		if ( matchingLeftNode != null ) {
			Token matchingLeftToken = matchingLeftNode.getSymbol();
			int i = matchingLeftToken.getTokenIndex();
			Token tokenAfterMatchingToken = doc.tokens.getNextRealToken(i);
//			System.out.printf("doc=%s node=%s, pair=%s, after=%s\n",
//			                  new File(doc.fileName).getName(), node.getSymbol(), matchingLeftToken, tokenAfterMatchingToken);
			if ( tokenAfterMatchingToken!=null ) {
				if ( tokenAfterMatchingToken.getType()==Token.EOF ) {
					return 1;
				}
				return tokenAfterMatchingToken.getLine()>matchingLeftToken.getLine() ? 1 : 0;
			}
		}
		return NOT_PAIR;
	}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
public static void wipeCharPositionInfoAndWhitespaceTokens(CodeBuffTokenStream tokens) {
	tokens.fill();
	CommonToken dummy = new CommonToken(Token.INVALID_TYPE, "");
	dummy.setChannel(Token.HIDDEN_CHANNEL);
	Token firstRealToken = tokens.getNextRealToken(-1);
	for (int i = 0; i<tokens.size(); i++) {
		if ( i==firstRealToken.getTokenIndex() ) continue; // don't wack first token
		CommonToken t = (CommonToken)tokens.get(i);
		if ( t.getText().matches("\\s+") ) {
			tokens.getTokens().set(i, dummy); // wack whitespace token so we can't use it during prediction
		}
		else {
			t.setLine(0);
			t.setCharPositionInLine(-1);
		}
	}
}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
@Override
public void exitDeclaration(ObjCParser.DeclarationContext ctx) {
    String decl = getCode(ctx.declaration_minus_semi()) + ";\n";

    //获取注释
    Token prior = ctx.getStart();
    int indx = prior.getTokenIndex();
    List<Token> cmtChannel = tokens.getHiddenTokensToLeft(indx,
            ObjCLexer.COMMENT_CHANNEL);
    if (cmtChannel != null) {
        String txt = "";
        for (Token cmt : cmtChannel) {
            if (cmt != null) {
                txt = txt + cmt.getText() + "\n";
            }
        }
        setComment(ctx, txt);
    }
    if (getComment(ctx) != null) {
        decl = getComment(ctx) + decl;
    }

    setCode(ctx, decl);
}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
@Override
public void exitInstance_method_definition(
        ObjCParser.Instance_method_definitionContext ctx) {
    String code = ctx.getText();
    String.format("s", code);
    String call = getCode(ctx.method_definition());
    String mDef = call;

    //获取注释
    Token prior = ctx.getStart();
    int indx = prior.getTokenIndex();
    List<Token> cmtChannel = tokens.getHiddenTokensToLeft(indx,
            ObjCLexer.COMMENT_CHANNEL);
    if (cmtChannel != null) {
        String txt = "";
        for (Token cmt : cmtChannel) {
            if (cmt != null) {
                txt = txt + cmt.getText() + "\n";
            }
        }
        setComment(ctx, txt);
    }
    if (getComment(ctx) != null) {
        mDef = getComment(ctx) + mDef;
    }

    setCode(ctx, mDef);
}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
@Override
public void exitExpression(ObjCParser.ExpressionContext ctx) {
    String message = ctx.getText();
    String.format("%s", message);
    String exp = getCode(ctx.assignment_expression().get(0));

    for (int i = 1; i < ctx.assignment_expression().size(); i++) {
        exp += ", " + getCode(ctx.assignment_expression().get(i));
    }

    //获取注释
    Token prior = ctx.getStart();
    int indx = prior.getTokenIndex();
    List<Token> cmtChannel = tokens.getHiddenTokensToLeft(indx,
            ObjCLexer.COMMENT_CHANNEL);
    if (cmtChannel != null) {
        String txt = "";
        for (Token cmt : cmtChannel) {
            if (cmt != null) {
                txt = txt + cmt.getText() + "\n";
            }
        }
        setComment(ctx, txt);
    }
    if (getComment(ctx) != null) {
        exp = getComment(ctx) + exp;
    }
    if (message == "[") {
        setCode(ctx, "[");
    } else if (message == "]") {
        setCode(ctx, "]");
    } else {
        setCode(ctx, exp);
    }
}
 

import org.antlr.v4.runtime.Token; //导入方法依赖的package包/类
public static SyntaxElement create(Token t, SyntaxElementType type) {
	return new SyntaxElement(t.getText(), t.getStartIndex(), t.getStopIndex(), t.getTokenIndex(), type);
}