Java TokenStream.get方法代码示例

import org.antlr.v4.runtime.TokenStream; //导入方法依赖的package包/类
/**
 "If an operator has whitespace on the right side only, it is treated as a
 postfix unary operator. As an example, the ++ operator in a++ b is treated
 as a postfix unary operator."
 "If an operator has no whitespace on the left but is followed immediately
 by a dot (.), it is treated as a postfix unary operator. As an example,
 the ++ operator in a++.b is treated as a postfix unary operator (a++ .b
 rather than a ++ .b)."
 */
public static boolean isPostfixOp(TokenStream tokens) {
	int stop = getLastOpTokenIndex(tokens);
	if ( stop==-1 ) return false;

	int start = tokens.index();
	Token prevToken = tokens.get(start-1); // includes hidden-channel tokens
	Token nextToken = tokens.get(stop+1);
	boolean prevIsWS = isLeftOperatorWS(prevToken);
	boolean nextIsWS = isRightOperatorWS(nextToken);
	boolean result =
		!prevIsWS && nextIsWS ||
		!prevIsWS && nextToken.getType()==SwiftParser.DOT;
	String text = tokens.getText(Interval.of(start, stop));
	//System.out.println("isPostfixOp: '"+prevToken+"','"+text+"','"+nextToken+"' is "+result);
	return result;
}
 

import org.antlr.v4.runtime.TokenStream; //导入方法依赖的package包/类
@NotNull
private String getText(TokenStream tokens, Interval interval)
{
    int start = interval.a;
    int stop = interval.b;
    if (start < 0 || stop < 0)
        return "";
    
    if (stop >= tokens.size())
        stop = tokens.size() - 1;
    
    StringBuilder buf = new StringBuilder();
    for (int i = start; i <= stop; i++)
    {
        Token t = tokens.get(i);
        if (t.getType() == Token.EOF)
            break;
        buf.append(t.getText());
        if (i != stop)
        {
            buf.append(" ");
        }
    }
    return buf.toString();
}
 

import org.antlr.v4.runtime.TokenStream; //导入方法依赖的package包/类
public Token addDecisionEventHighlighter(PreviewState previewState, MarkupModel markupModel,
                                         DecisionEventInfo info, Color errorStripeColor,
                                         EffectType effectType) {
	TokenStream tokens = previewState.parsingResult.parser.getInputStream();
	Token startToken = tokens.get(info.startIndex);
	Token stopToken = tokens.get(info.stopIndex);
	TextAttributes textAttributes =
		new TextAttributes(JBColor.BLACK, JBColor.WHITE, errorStripeColor,
		                   effectType, Font.PLAIN);
	textAttributes.setErrorStripeColor(errorStripeColor);
	final RangeHighlighter rangeHighlighter =
		markupModel.addRangeHighlighter(
			startToken.getStartIndex(), stopToken.getStopIndex()+1,
			HighlighterLayer.ADDITIONAL_SYNTAX, textAttributes,
			HighlighterTargetArea.EXACT_RANGE);
	rangeHighlighter.putUserData(DECISION_EVENT_INFO_KEY, info);
	rangeHighlighter.setErrorStripeMarkColor(errorStripeColor);
	return startToken;
}
 

import org.antlr.v4.runtime.TokenStream; //导入方法依赖的package包/类
public static boolean isOpNext(TokenStream tokens) {
	int start = tokens.index();
	Token lt = tokens.get(start);
	int stop = getLastOpTokenIndex(tokens);
	if ( stop==-1 ) return false;
	System.out.printf("isOpNext: i=%d t='%s'", start, lt.getText());
	System.out.printf(", op='%s'\n", tokens.getText(Interval.of(start,stop)));
	return true;
}
 

import org.antlr.v4.runtime.TokenStream; //导入方法依赖的package包/类
/**
 "If an operator has whitespace around both sides or around neither side,
 it is treated as a binary operator. As an example, the + operator in a+b
 and a + b is treated as a binary operator."
 */
public static boolean isBinaryOp(TokenStream tokens) {
	int stop = getLastOpTokenIndex(tokens);
	if ( stop==-1 ) return false;

	int start = tokens.index();
	Token prevToken = tokens.get(start-1); // includes hidden-channel tokens
	Token nextToken = tokens.get(stop+1);
	boolean prevIsWS = isLeftOperatorWS(prevToken);
	boolean nextIsWS = isRightOperatorWS(nextToken);
	boolean result = prevIsWS && nextIsWS || (!prevIsWS && !nextIsWS);
	String text = tokens.getText(Interval.of(start, stop));
	//System.out.println("isBinaryOp: '"+prevToken+"','"+text+"','"+nextToken+"' is "+result);
	return result;
}
 

import org.antlr.v4.runtime.TokenStream; //导入方法依赖的package包/类
/**
 "If an operator has whitespace on the left side only, it is treated as a
 prefix unary operator. As an example, the ++ operator in a ++b is treated
 as a prefix unary operator."
*/
public static boolean isPrefixOp(TokenStream tokens) {
	int stop = getLastOpTokenIndex(tokens);
	if ( stop==-1 ) return false;

	int start = tokens.index();
	Token prevToken = tokens.get(start-1); // includes hidden-channel tokens
	Token nextToken = tokens.get(stop+1);
	boolean prevIsWS = isLeftOperatorWS(prevToken);
	boolean nextIsWS = isRightOperatorWS(nextToken);
	boolean result = prevIsWS && !nextIsWS;
	String text = tokens.getText(Interval.of(start, stop));
	//System.out.println("isPrefixOp: '"+prevToken+"','"+text+"','"+nextToken+"' is "+result);
	return result;
}
 

import org.antlr.v4.runtime.TokenStream; //导入方法依赖的package包/类
/** Return two booleans packed into lowest 2 bits for left (high) and right (low)
 *  whitespace.
 */
public static int getLeftRightWS(TokenStream tokens, ParserRuleContext ctx) {
	int left = ctx.start.getTokenIndex();
	int right = ctx.stop.getTokenIndex();
	Token prevToken = tokens.get(left-1); // includes hidden-channel tokens
	Token nextToken = tokens.get(right+1);
	boolean prevIsWS = isLeftOperatorWS(prevToken);
	boolean nextIsWS = isRightOperatorWS(nextToken);
	int b = (prevIsWS ? 1 : 0) << 1 | (nextIsWS ? 1 : 0) ;
	return b;
}
 

import org.antlr.v4.runtime.TokenStream; //导入方法依赖的package包/类
/**
 * "If an operator has whitespace around both sides or around neither side,
 * it is treated as a binary operator. As an example, the + operator in a+b
 * and a + b is treated as a binary operator."
 */
public static boolean isBinaryOp(TokenStream tokens) {
    int stop = getLastOpTokenIndex(tokens);
    if (stop == -1) return false;

    int start = tokens.index();
    Token prevToken = tokens.get(start - 1); // includes hidden-channel tokens
    Token nextToken = tokens.get(stop + 1);
    boolean prevIsWS = isLeftOperatorWS(prevToken);
    boolean nextIsWS = isRightOperatorWS(nextToken);
    return prevIsWS && nextIsWS || (!prevIsWS && !nextIsWS);
}
 

import org.antlr.v4.runtime.TokenStream; //导入方法依赖的package包/类
/**
 * "If an operator has whitespace on the left side only, it is treated as a
 * prefix unary operator. As an example, the ++ operator in a ++b is treated
 * as a prefix unary operator."
 */
public static boolean isPrefixOp(TokenStream tokens) {
    int stop = getLastOpTokenIndex(tokens);
    if (stop == -1) return false;

    int start = tokens.index();
    Token prevToken = tokens.get(start - 1); // includes hidden-channel tokens
    Token nextToken = tokens.get(stop + 1);
    boolean prevIsWS = isLeftOperatorWS(prevToken);
    boolean nextIsWS = isRightOperatorWS(nextToken);
    return prevIsWS && !nextIsWS;
}
 

import org.antlr.v4.runtime.TokenStream; //导入方法依赖的package包/类
/**
 * "If an operator has whitespace on the right side only, it is treated as a
 * postfix unary operator. As an example, the ++ operator in a++ b is treated
 * as a postfix unary operator."
 * <p>
 * "If an operator has no whitespace on the left but is followed immediately
 * by a dot (.), it is treated as a postfix unary operator. As an example,
 * the ++ operator in a++.b is treated as a postfix unary operator (a++ .b
 * rather than a ++ .b)."
 */
public static boolean isPostfixOp(TokenStream tokens) {
    int stop = getLastOpTokenIndex(tokens);
    if (stop == -1) return false;

    int start = tokens.index();
    Token prevToken = tokens.get(start - 1); // includes hidden-channel tokens
    Token nextToken = tokens.get(stop + 1);
    boolean prevIsWS = isLeftOperatorWS(prevToken);
    boolean nextIsWS = isRightOperatorWS(nextToken);
    return
            !prevIsWS && nextIsWS ||
                    !prevIsWS && nextToken.getType() == SwiftParser.DOT;
}
 

import org.antlr.v4.runtime.TokenStream; //导入方法依赖的package包/类
@NotNull
protected NoViableAltException noViableAlt(@NotNull TokenStream input,
										@NotNull ParserRuleContext outerContext,
										@NotNull ATNConfigSet configs,
										int startIndex)
{
	return new NoViableAltException(parser, input,
										input.get(startIndex),
										input.LT(1),
										configs, outerContext);
}
 

import org.antlr.v4.runtime.TokenStream; //导入方法依赖的package包/类
public void setProfilerData(PreviewState previewState, long parseTime_ns) {
	this.previewState = previewState;
	Parser parser = previewState.parsingResult.parser;
	ParseInfo parseInfo = parser.getParseInfo();
	updateTableModelPerExpertCheckBox(parseInfo);
	long parseTimeMS = (long) (parseTime_ns/(1000.0*1000.0));
	parseTimeField.setText(String.valueOf(parseTimeMS));
	int predTimeMS = (int) (parseInfo.getTotalTimeInPrediction()/(1000.0*1000.0));
	predictionTimeField.setText(
		String.format("%d = %3.2f%%", predTimeMS, 100*((double) predTimeMS)/parseTimeMS)
	                           );
	TokenStream tokens = parser.getInputStream();
	int numTokens = tokens.size();
	Token lastToken = tokens.get(numTokens-1);
	int numChar = lastToken.getStopIndex();
	int numLines = lastToken.getLine();
	if ( lastToken.getType()==Token.EOF ) {
		if ( numTokens<=1 ) {
			numLines = 0;
		}
		else {
			Token secondToLastToken = tokens.get(numTokens-2);
			numLines = secondToLastToken.getLine();
		}
	}
	inputSizeField.setText(String.format("%d char, %d lines",
	                                     numChar,
	                                     numLines));
	numTokensField.setText(String.valueOf(numTokens));
	double look =
		parseInfo.getTotalSLLLookaheadOps()+
			parseInfo.getTotalLLLookaheadOps();
	lookaheadBurdenField.setText(
		String.format("%d/%d = %3.2f", (long) look, numTokens, look/numTokens)
	                            );
	double atnLook = parseInfo.getTotalATNLookaheadOps();
	cacheMissRateField.setText(
		String.format("%d/%d = %3.2f%%", (long) atnLook, (long) look, atnLook*100.0/look)
	                          );
}
 

import org.antlr.v4.runtime.TokenStream; //导入方法依赖的package包/类
public static Token getTokenForCharIndex(TokenStream tokens, int charIndex) {
	for (int i=0; i<tokens.size(); i++) {
		Token t = tokens.get(i);
		if ( charIndex>=t.getStartIndex() && charIndex<=t.getStopIndex() ) {
			return t;
		}
	}
	return null;
}
 

import org.antlr.v4.runtime.TokenStream; //导入方法依赖的package包/类
@Override
public void actionPerformed(AnActionEvent e) {
	PsiElement el = MyActionUtils.getSelectedPsiElement(e);
	if ( el==null ) return;

	final PsiFile psiFile = e.getData(LangDataKeys.PSI_FILE);
	if (psiFile == null) return;

	Editor editor = e.getData(PlatformDataKeys.EDITOR);
	if ( editor==null ) return;
	final Document doc = editor.getDocument();
	SelectionModel selectionModel = editor.getSelectionModel();

	String grammarText = psiFile.getText();
	ParsingResult results = ParsingUtils.parseANTLRGrammar(grammarText);
	final Parser parser = results.parser;
	final ParserRuleContext tree = (ParserRuleContext)results.tree;
	TokenStream tokens = parser.getTokenStream();

	int selStart = selectionModel.getSelectionStart();
	int selStop = selectionModel.getSelectionEnd()-1; // I'm inclusive and they are exclusive for end offset

	// find appropriate tokens for bounds, don't include WS
	Token start = RefactorUtils.getTokenForCharIndex(tokens, selStart);
	Token stop = RefactorUtils.getTokenForCharIndex(tokens, selStop);
	if ( start==null || stop==null ) {
		return;
	}
	if ( start.getType()==ANTLRv4Lexer.WS ) {
		start = tokens.get(start.getTokenIndex()+1);
	}
	if ( stop.getType()==ANTLRv4Lexer.WS ) {
		stop = tokens.get(stop.getTokenIndex()-1);
	}

	selectionModel.setSelection(start.getStartIndex(), stop.getStopIndex() + 1);
	final Project project = e.getProject();
	final ChooseExtractedRuleName nameChooser = new ChooseExtractedRuleName(project);
	nameChooser.show();
	if ( nameChooser.ruleName==null ) return;

	// make new rule string
	final String ruleText = selectionModel.getSelectedText();

	final int insertionPoint = RefactorUtils.getCharIndexOfNextRuleStart(tree, start.getTokenIndex());
	final String newRule = "\n"+nameChooser.ruleName + " : " + ruleText + " ;" + "\n";

	final Token start_ = start;
	final Token stop_ = stop;
	WriteCommandAction setTextAction = new WriteCommandAction(project) {
		@Override
		protected void run(final Result result) throws Throwable {
			// do all as one operation.
			if ( insertionPoint>=doc.getTextLength() ) {
				doc.insertString(doc.getTextLength(), newRule);
			}
			else {
				doc.insertString(insertionPoint, newRule);
			}
			doc.replaceString(start_.getStartIndex(), stop_.getStopIndex()+1, nameChooser.ruleName);
		}
	};
	setTextAction.execute();

	// TODO: only allow selection of fully-formed syntactic entity.
	// E.g., "A (',' A" is invalid grammatically as a rule.
}