Java Token类代码示例

import com.sonar.sslr.api.Token; //导入依赖的package包/类
@Override
public void visitToken(Token token) {
  if (token.isGeneratedCode()) {
    return;
  }

  if (previousToken != null && previousToken.getLine() != token.getLine()) {
    // Note that AbstractLineLengthCheck doesn't support tokens which span multiple lines - see SONARPLUGINS-2025
    String[] lines = previousToken.getValue().split("\r?\n|\r", -1);
    int length = previousToken.getColumn();
    for (String line : lines) {
      length += line.length();
      if (length > getMaximumLineLength()) {
        // Note that method from AbstractLineLengthCheck generates other message - see SONARPLUGINS-1809
        getContext().createLineViolation(this,
          "The line contains {0,number,integer} characters which is greater than {1,number,integer} authorized.",
          previousToken.getLine(),
          length,
          getMaximumLineLength());
      }
      length = 0;
    }
  }
  previousToken = token;
}
 

import com.sonar.sslr.api.Token; //导入依赖的package包/类
@Override
public void visitToken(Token token) {
  if (token.getType().equals(PuppetTokenType.SINGLE_QUOTED_STRING_LITERAL) || token.getType().equals(PuppetTokenType.DOUBLE_QUOTED_STRING_LITERAL)) {
    // case: string literal, including doc string
    highlight(token, TypeOfText.STRING);

  } else if (token.getType() instanceof PuppetKeyword) {
    // case: keyword
    highlight(token, TypeOfText.KEYWORD);
  }

  for (Trivia trivia : token.getTrivia()) {
    // case: comment
    highlight(trivia.getToken(), TypeOfText.COMMENT);
  }
}
 

import com.sonar.sslr.api.Token; //导入依赖的package包/类
public TokenLocation(Token token) {
  this.startLine = token.getLine();
  this.startLineOffset = token.getColumn();

  String value = token.getValue();
  String[] lines = value.split("\r\n|\n|\r", -1);

  if (lines.length > 1) {
    endLine = token.getLine() + lines.length - 1;
    endLineOffset = lines[lines.length - 1].length();

  } else {
    this.endLine = this.startLine;
    this.endLineOffset = this.startLineOffset + token.getValue().length();
  }
}
 

import com.sonar.sslr.api.Token; //导入依赖的package包/类
@Override
public boolean consume(CodeReader code, Lexer lexer) {
  if (code.popTo(matcher, tmpBuilder) > 0) {
    String word = tmpBuilder.toString();
    String wordOriginal = word;
    if (!caseSensitive) {
      word = word.toUpperCase();
    }

    TokenType keywordType = keywordsMap.get(word);
    Token token = tokenBuilder
      .setType(keywordType == null ? NAME : keywordType)
      .setValueAndOriginalValue(word, wordOriginal)
      .setURI(lexer.getURI())
      .setLine(code.getPreviousCursor().getLine())
      .setColumn(code.getPreviousCursor().getColumn())
      .build();

    lexer.addToken(token);

    tmpBuilder.delete(0, tmpBuilder.length());
    return true;
  }
  return false;
}
 

import com.sonar.sslr.api.Token; //导入依赖的package包/类
public void visitToken(Token token) {
  if (!token.isGeneratedCode()) {
    if (previousToken != null && previousToken.getLine() != token.getLine()) {
      int length = previousToken.getColumn() + previousToken.getValue().length();
      if (length > getMaximumLineLength()) {
        // Note that method from AbstractLineLengthCheck generates other message - see SONARPLUGINS-1809
        getContext().createLineViolation(this,
            "The line contains {0,number,integer} characters which is greater than {1,number,integer} authorized.",
            previousToken.getLine(),
            length,
            getMaximumLineLength());
      }
    }
    previousToken = token;
  }
}
 

import com.sonar.sslr.api.Token; //导入依赖的package包/类
@Override
public void visitToken(Token token) {

    if (token.getType().equals(GenericTokenType.EOF)) {
        return;
    }
    
    if(token.getType() == GenericTokenType.UNKNOWN_CHAR){
        linesOfCode.add(token.getLine());
    }

    if(token.getType() == GenericTokenType.COMMENT){
        linesOfComments.add(token.getLine());
    }
}
 

import com.sonar.sslr.api.Token; //导入依赖的package包/类
/**
 * Builds and stores a {@link TokenEntry} from the source code.
 *
 * @param source source code.
 * @param cpdTokens tokens.
 * @throws IOException when it can't read a source file.
 */
@Override
public void tokenize(SourceCode source, Tokens cpdTokens) throws IOException {
    Lexer lexer = ApexLexer.create(config);
    String fileName = source.getFileName();
    List<Token> tokens = lexer.lex(new File(fileName));
    tokens.forEach(token -> {
        TokenEntry cpdToken = new TokenEntry(getTokenImage(token), fileName, token.getLine());
        cpdTokens.add(cpdToken);
    });
    cpdTokens.add(TokenEntry.getEOF());
}
 

import com.sonar.sslr.api.Token; //导入依赖的package包/类
public void startElement(String uri, String name, String qName, Attributes atts) {
	int line = locator.getLineNumber();
	int column = locator.getColumnNumber();
	if(FlowLexer.FlowTypes.isInEnum("START_" + name.toUpperCase())){
		logger.debug("Start element: " + qName + "[" + line + "," + column + "]" + "[START_" + name.toUpperCase() + "]");
		Token token = tokenBuilder.setType(FlowLexer.FlowTypes.valueOf("START_" + name.toUpperCase())).setValueAndOriginalValue(name.toUpperCase(),name)
				.setURI(lex.getURI()).setLine(line).setColumn(0).build();
		lex.addToken(token);
		// CHECK THE ATTRIBUTES
		for(int i=0; i<atts.getLength();i++){
			if(FlowLexer.FlowAttTypes.isInEnum(atts.getQName(i).toUpperCase())){
				token = tokenBuilder.setType(FlowLexer.FlowAttTypes.getEnum(atts.getQName(i).toUpperCase())).setValueAndOriginalValue(atts.getValue(i).toUpperCase(),atts.getValue(i))
						.setURI(lex.getURI()).setLine(line).setColumn(0).build();
				lex.addToken(token);
				logger.debug("TOKEN " + token.getValue() + "[" + token.getLine() + "," + token.getColumn() + "]");
			}else{
				token = tokenBuilder.setType(GenericTokenType.IDENTIFIER).setValueAndOriginalValue(atts.getQName(i))
						.setURI(lex.getURI()).setLine(line).setColumn(0).build();
				lex.addToken(token);
				logger.debug("IDENTIFIER " + token.getValue() + "[" + token.getLine() + "," + token.getColumn() + "]");
				token = tokenBuilder.setType(GenericTokenType.LITERAL).setValueAndOriginalValue(atts.getValue(i))
						.setURI(lex.getURI()).setLine(line).setColumn(0).build();
				lex.addToken(token);
				logger.debug("LITERAL " + token.getValue() + "[" + token.getLine() + "," + token.getColumn() + "]");
			}
		}
	}
}
 

import com.sonar.sslr.api.Token; //导入依赖的package包/类
@Override
public void visitToken(Token token) {
  Iterator iterator = token.getTrivia().iterator();
  while (iterator.hasNext()) {
    Trivia trivia = (Trivia) iterator.next();
    if (trivia.isComment() && pattern.matcher(trivia.getToken().getOriginalValue()).matches()) {
      addIssue(trivia.getToken().getLine(), this, "Use starting comment token '#' instead.");
    }
  }
}
 

import com.sonar.sslr.api.Token; //导入依赖的package包/类
public void visitToken(Token token) {
  for (Trivia trivia : token.getTrivia()) {
    String comment = trivia.getToken().getOriginalValue();
    if (StringUtils.containsIgnoreCase(comment, pattern)) {
      String[] lines = comment.split("\r\n?|\n");

      for (int i = 0; i < lines.length; i++) {
        if (StringUtils.containsIgnoreCase(lines[i], pattern) && !isLetterAround(lines[i], pattern)) {
          check.addIssue(trivia.getToken().getLine() + i, check, message);
        }
      }
    }
  }
}
 

import com.sonar.sslr.api.Token; //导入依赖的package包/类
@Override
public final void tokenize(SourceCode source, Tokens cpdTokens) {
  Lexer lexer = PuppetLexer.create(new PuppetConfiguration(charset));
  String fileName = source.getFileName();
  List<Token> tokens = lexer.lex(new File(fileName));
  for (Token token : tokens) {
    if (!token.getType().equals(PuppetTokenType.NEWLINE) && !token.getType().equals(PuppetTokenType.DEDENT) && !token.getType().equals(PuppetTokenType.INDENT)) {
      TokenEntry cpdToken = new TokenEntry(getTokenImage(token), fileName, token.getLine());
      cpdTokens.add(cpdToken);
    }
  }
  cpdTokens.add(TokenEntry.getEOF());
}
 

import com.sonar.sslr.api.Token; //导入依赖的package包/类
/**
 * {@inheritDoc}
 */
@Override
public void visitToken(Token token) {
  if (!token.getType().equals(EOF)) {
    String[] tokenLines = token.getValue().split("\n", -1);

    int firstLineAlreadyCounted = lastTokenLine == token.getLine() ? 1 : 0;
    getContext().peekSourceCode().add(metric, (double) tokenLines.length - firstLineAlreadyCounted);

    lastTokenLine = token.getLine() + tokenLines.length - 1;
  }

}
 

import com.sonar.sslr.api.Token; //导入依赖的package包/类
@Override
public boolean consume(CodeReader code, Lexer output) {
  if (code.peek() == '/') {
    Token lastToken = getLastToken(output);
    if (lastToken == null || lastToken.getType().equals(PuppetKeyword.NODE) || guessNextIsRegexp(lastToken.getValue())) {
      return delegate.consume(code, output);
    }
  }
  return false;
}
 

import com.sonar.sslr.api.Token; //导入依赖的package包/类
@Override
public void matchSuccess(final MatchSuccessEvent<Token.Builder> event)
{
    final MatcherContext<Token.Builder> context = event.getContext();
    if (!context.inPredicate())
        consumed = Math.max(consumed, context.getCurrentIndex());
    if (context.getLevel() != 0)
        return;
    if (context.getMatcher() != rootMatcher)
        throw new IllegalStateException("was expecting root rule here");
}
 

import com.sonar.sslr.api.Token; //导入依赖的package包/类
@Override
public void matchFailure(final MatchFailureEvent<Token.Builder> event)
{
    final MatcherContext<Token.Builder> context = event.getContext();
    if (context.getLevel() != 0)
        return;
    if (context.getMatcher() != rootMatcher)
        throw new IllegalStateException("was expecting root rule here");
}