Java CommonToken.setCharPositionInLine方法代码示例

import org.antlr.v4.runtime.CommonToken; //导入方法依赖的package包/类
public static void wipeCharPositionInfoAndWhitespaceTokens(CodeBuffTokenStream tokens) {
	tokens.fill();
	CommonToken dummy = new CommonToken(Token.INVALID_TYPE, "");
	dummy.setChannel(Token.HIDDEN_CHANNEL);
	Token firstRealToken = tokens.getNextRealToken(-1);
	for (int i = 0; i<tokens.size(); i++) {
		if ( i==firstRealToken.getTokenIndex() ) continue; // don't wack first token
		CommonToken t = (CommonToken)tokens.get(i);
		if ( t.getText().matches("\\s+") ) {
			tokens.getTokens().set(i, dummy); // wack whitespace token so we can't use it during prediction
		}
		else {
			t.setLine(0);
			t.setCharPositionInLine(-1);
		}
	}
}
 

import org.antlr.v4.runtime.CommonToken; //导入方法依赖的package包/类
@Override
	public void syntaxError(Recognizer<?, ?> recognizer,
							Object offendingSymbol,
							int line,
							int charPositionInLine,
							String msg,
							RecognitionException e)
	{
		if ( offendingSymbol==null ) {
			final Lexer lexer = (Lexer) recognizer;
			int i = lexer.getCharIndex();
			final int n = lexer.getInputStream().size();
			if (i >= n) {
				i = n - 1;
			}
			final String text = lexer.getInputStream().getText(new Interval(i, i));
			CommonToken t = (CommonToken) lexer.getTokenFactory().create(Token.INVALID_TYPE, text);
			t.setStartIndex(i);
			t.setStopIndex(i);
			t.setLine(line);
			t.setCharPositionInLine(charPositionInLine);
			offendingSymbol = t;
		}
//		System.out.println("lex error: " + offendingSymbol);
		issues.add(new Issue(msg, (Token)offendingSymbol));
	}
 

import org.antlr.v4.runtime.CommonToken; //导入方法依赖的package包/类
private void addToken(String prefix, String s, TokenType tokenType) {
  CommonToken token = new CommonToken(tokenType.ordinal(), prefix + s);
  token.setCharPositionInLine(pos);
  token.setLine(lineNo);
  pos += s.length();
  builder.add(token);
}
 

import org.antlr.v4.runtime.CommonToken; //导入方法依赖的package包/类
public void processToken(int indexIntoRealTokens, int tokenIndexInStream, boolean collectAnalysis) {
	CommonToken curToken = (CommonToken)testDoc.tokens.get(tokenIndexInStream);
	String tokText = curToken.getText();
	TerminalNode node = tokenToNodeMap.get(curToken);

	int[] features = getFeatures(testDoc, tokenIndexInStream);
	int[] featuresForAlign = new int[features.length];
	System.arraycopy(features, 0, featuresForAlign, 0, features.length);

	int injectNL_WS = wsClassifier.classify(k, features, Trainer.MAX_WS_CONTEXT_DIFF_THRESHOLD);

	injectNL_WS = emitCommentsToTheLeft(tokenIndexInStream, injectNL_WS);

	int newlines = 0;
	int ws = 0;
	if ( (injectNL_WS&0xFF)==CAT_INJECT_NL ) {
		newlines = Trainer.unnlcat(injectNL_WS);
	}
	else if ( (injectNL_WS&0xFF)==CAT_INJECT_WS ) {
		ws = Trainer.unwscat(injectNL_WS);
	}

	if ( newlines==0 && ws==0 && cannotJoin(realTokens.get(indexIntoRealTokens-1), curToken) ) { // failsafe!
		ws = 1;
	}

	int alignOrIndent = CAT_ALIGN;

	if ( newlines>0 ) {
		output.append(Tool.newlines(newlines));
		line+=newlines;
		charPosInLine = 0;

		// getFeatures() doesn't know what line curToken is on. If \n, we need to find exemplars that start a line
		featuresForAlign[INDEX_FIRST_ON_LINE] = 1; // use \n prediction to match exemplars for alignment

		alignOrIndent = hposClassifier.classify(k, featuresForAlign, MAX_ALIGN_CONTEXT_DIFF_THRESHOLD);

		if ( (alignOrIndent&0xFF)==CAT_ALIGN_WITH_ANCESTOR_CHILD ) {
			align(alignOrIndent, node);
		}
		else if ( (alignOrIndent&0xFF)==CAT_INDENT_FROM_ANCESTOR_CHILD ) {
			indent(alignOrIndent, node);
		}
		else if ( (alignOrIndent&0xFF)==CAT_ALIGN ) {
			List<Token> tokensOnPreviousLine = getTokensOnPreviousLine(testDoc.tokens, tokenIndexInStream, line);
			if ( tokensOnPreviousLine.size()>0 ) {
				Token firstTokenOnPrevLine = tokensOnPreviousLine.get(0);
				int indentCol = firstTokenOnPrevLine.getCharPositionInLine();
				charPosInLine = indentCol;
				output.append(Tool.spaces(indentCol));
			}
		}
		else if ( (alignOrIndent&0xFF)==CAT_INDENT ) {
			indent(alignOrIndent, node);
		}
	}
	else {
		// inject whitespace instead of \n?
		output.append(Tool.spaces(ws));
		charPosInLine += ws;
	}

	// update Token object with position information now that we are about
	// to emit it.
	curToken.setLine(line);
	curToken.setCharPositionInLine(charPosInLine);

	TokenPositionAnalysis tokenPositionAnalysis =
		getTokenAnalysis(features, featuresForAlign, tokenIndexInStream, injectNL_WS, alignOrIndent, collectAnalysis);

	analysis.set(tokenIndexInStream, tokenPositionAnalysis);

	int n = tokText.length();
	tokenPositionAnalysis.charIndexStart = output.length();
	tokenPositionAnalysis.charIndexStop = tokenPositionAnalysis.charIndexStart + n - 1;

	// emit
	output.append(tokText);
	charPosInLine += n;
}