Java Token類代碼示例

import org.eclipse.jface.text.rules.Token; //導入依賴的package包/類
@Override
public IToken nextToken() {
	if (currentTemplateTextToken != null) {
		if (currentTemplateTextToken.hasNext())
			return currentTemplateTextToken.nextToken();
		else
			currentTemplateTextToken = null;
	}
	if (!getIterator().hasNext())
		return Token.EOF;
	ILexerTokenRegion next = getIterator().next();
	int tokenType = next.getLexerTokenType();
	switch (tokenType) {
	case RULE_TEMPLATE_HEAD:
	case RULE_TEMPLATE_MIDDLE: {
		currentTemplateTextToken = createTemplateToken(tokenType, next);
		return currentTemplateTextToken.nextToken();
	}
	default:
		setCurrentToken(next);
		return createToken(next);
	}
}
 

import org.eclipse.jface.text.rules.Token; //導入依賴的package包/類
@Override
public IToken evaluate(ICharacterScanner scanner1, boolean resume) {
	if (!(scanner1 instanceof BatchDocumentPartitionScanner)){
		return Token.UNDEFINED;
	}
	BatchDocumentPartitionScanner scanner = (BatchDocumentPartitionScanner)scanner1;
	if (scanner.getOffset()<ECHO.length()){
		/* would be a problem on start on going back...*/
		return Token.UNDEFINED;
	}
	scanner.unread(); // go to character before
	int charBeforeAsInt = scanner.read(); // get character, moves to origin pos again
	/* fast guard closing:*/
	if (charBeforeAsInt!='o' && charBeforeAsInt!='O'){ // echo ECHO <- last character must be an o other wise guard close...
		return Token.UNDEFINED;
	}
	ICharacterScannerCodePosSupport codePosSupport = new ICharacterScannerCodePosSupport(scanner);
	if (parserSupport.isAfterEchoHandled(codePosSupport)){
		return getSuccessToken();
	}
	codePosSupport.resetToStartPos();
	return Token.UNDEFINED;
}
 

import org.eclipse.jface.text.rules.Token; //導入依賴的package包/類
@Override
public IToken evaluate(ICharacterScanner scanner, boolean resume) {
	int r = scanner.read();
	scanner.unread();
	/* fast guard closing:*/
	if (ICharacterScanner.EOF==r || r!='<'){
		return Token.UNDEFINED;
	}
	
	ICharacterScannerCodePosSupport codePosSupport = new ICharacterScannerCodePosSupport(scanner);
	if (hereStringSupport.isHereStringStateHandled(codePosSupport)){
		return getSuccessToken();
	}
	codePosSupport.resetToStartPos();
	return Token.UNDEFINED;
}
 

import org.eclipse.jface.text.rules.Token; //導入依賴的package包/類
@Override
public IToken evaluate(ICharacterScanner scanner, boolean resume) {
	int r = scanner.read();
	scanner.unread();
	/* fast guard closing:*/
	if (ICharacterScanner.EOF==r || r!='<'){
		return Token.UNDEFINED;
	}
	
	ICharacterScannerCodePosSupport codePosSupport = new ICharacterScannerCodePosSupport(scanner);
	if (hereDocSupport.isHereDocStateHandled(codePosSupport)){
		return getSuccessToken();
	}
	codePosSupport.resetToStartPos();
	return Token.UNDEFINED;
}
 

import org.eclipse.jface.text.rules.Token; //導入依賴的package包/類
public RelationModelPartitionScanner() {
  final IToken commentPartition = new Token(RelationModelPartitionScanner.RELATION_MODEL_COMMENT);
  final IToken optionPartition = new Token(RelationModelPartitionScanner.RELATION_MODEL_OPTION);
  final IToken universePartition =
      new Token(RelationModelPartitionScanner.RELATION_MODEL_UNIVERSE);
  final IToken relBoundPartition =
      new Token(RelationModelPartitionScanner.RELATION_MODEL_REL_BOUND);

  final List<IRule> rules = new ArrayList<IRule>();
  rules.add(new MultiLineRule("/**", "**/", commentPartition));
  rules.add(new EndOfLineRule("--", commentPartition));
  // rules.add(new EndOfLineRule("//", commentPartition));
  rules.add(new MultiLineRule("options {", "}", optionPartition));
  rules.add(new MultiLineRule("universe {", "}", universePartition));
  rules.add(new MultiLineRule("universe [", "]", universePartition));
  rules.add(new RelationBoundsRule("relations", "}", relBoundPartition));

  final IPredicateRule[] result = new IPredicateRule[rules.size()];
  rules.toArray(result);
  this.setPredicateRules(result);
}
 

import org.eclipse.jface.text.rules.Token; //導入依賴的package包/類
@Override
protected IToken doEvaluate(final ICharacterScanner scanner, final boolean resume) {

  if (resume) {
    if (this.endSequenceDetected(scanner)) {
      return this.fToken;
    }
  } else {
    final int c = scanner.read();
    if (c == this.fStartSequence[0]) {
      if (this.sequenceDetected(scanner, this.fStartSequence, false)) {
        if (this.endSequenceDetected(scanner)) {
          return this.fToken;
        }
      }
    }
  }

  scanner.unread();
  return Token.UNDEFINED;
}
 

import org.eclipse.jface.text.rules.Token; //導入依賴的package包/類
public ModelScanner(final ColorManager colorManager) {

    final IToken bracketsToken =
        new Token(new TextAttribute(colorManager.getColor(IValidationColorConstants.BRACKETS)));
    final IToken commToken =
        new Token(new TextAttribute(colorManager.getColor(IValidationColorConstants.COMMENT)));

    final List<IRule> rules = new ArrayList<>();

    rules.add(new WhitespaceRule(new ValidationWhitespaceDetector()));
    rules.add(new BracketsRule(bracketsToken));
    rules.add(new MultiLineRule("/**", "**/", commToken));
    rules.add(new EndOfLineRule("--", commToken));

    final IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    this.setRules(result);
  }
 

import org.eclipse.jface.text.rules.Token; //導入依賴的package包/類
public EiffelPartitionScanner() {
	super();
	
	IToken multilineComment = new Token(MULTILINE_COMMENT);
	IToken singlelineComment = new Token(SINGLELINE_COMMENT);
	IToken string = new Token(STRING);
	
	List<Object> rules = new ArrayList<>();
	
	//Add rule for single line comments.
	rules.add(new EndOfLineRule("--", singlelineComment));
	
	// Add rule for strings and character constants.
	rules.add(new SingleLineRule("\"", "\"", string, '\\'));
	rules.add(new SingleLineRule("'", "'", string, '\\'));
	
	//Add rules for multi-line comments and eiffeldocs
	rules.add(new MultiLineRule("--[[", "]]", multilineComment, (char) 0, true));
	
	IPredicateRule[] result = new IPredicateRule[rules.size()];
	rules.toArray(result);
	setPredicateRules(result);
}
 

import org.eclipse.jface.text.rules.Token; //導入依賴的package包/類
public XMLTextScanner( ColorManager colorManager) {

      ESCAPED_CHAR = new Token(new TextAttribute(colorManager.getColor(IXMLColorConstants.ESCAPED_CHAR)));
      CDATA_START = new Token(new TextAttribute(colorManager.getColor(IXMLColorConstants.CDATA)));
      CDATA_END = new Token(new TextAttribute(colorManager.getColor(IXMLColorConstants.CDATA)));
      CDATA_TEXT = new Token(new TextAttribute(colorManager.getColor(IXMLColorConstants.CDATA_TEXT)));
      IRule[] rules = new IRule[2];

      // Add rule to pick up escaped chars
      // Add rule to pick up start of CDATA section
      rules[0] = new CDataRule(CDATA_START, true);
      // Add a rule to pick up end of CDATA sections
      rules[1] = new CDataRule(CDATA_END, false);
      setRules(rules);

   }
 

import org.eclipse.jface.text.rules.Token; //導入依賴的package包/類
public XMLPartitionScanner() {

      IToken xmlComment = new Token(XML_COMMENT);
      IToken xmlPI = new Token(XML_PI);
      IToken startTag = new Token(XML_START_TAG);
      IToken endTag = new Token(XML_END_TAG);
      IToken docType = new Token(XML_DOCTYPE);
      IToken text = new Token(XML_TEXT);

      IPredicateRule[] rules = new IPredicateRule[7];

      rules[0] = new NonMatchingRule();
      rules[1] = new MultiLineRule("<!--", "-->", xmlComment);
      rules[2] = new MultiLineRule("<?", "?>", xmlPI);
      rules[3] = new MultiLineRule("</", ">", endTag);
      rules[4] = new StartTagRule(startTag);
      rules[5] = new MultiLineRule("<!DOCTYPE", ">", docType);
      rules[6] = new XMLTextPredicateRule(text);

      setPredicateRules(rules);
   }
 

import org.eclipse.jface.text.rules.Token; //導入依賴的package包/類
public IToken evaluate( ICharacterScanner scanner){

      buffer.setLength(0);

      int c = read(scanner);
      if (c == '&') {

         int i = 0;
         do {
            c = read(scanner);
            i++;

            if (c == '<' || c == ']') {
               for (int j = i - 1; j > 0; j--)
                  scanner.unread();
               return Token.UNDEFINED;
            }
         } while (c != ';');
         return fToken;
      }

      scanner.unread();
      return Token.UNDEFINED;
   }
 

import org.eclipse.jface.text.rules.Token; //導入依賴的package包/類
public IToken evaluate( ICharacterScanner scanner){

      buffer.setLength(0);

      charsRead = 0;
      int c = read(scanner);

      if (c == matchString.charAt(0)) {
         do {
            c = read(scanner);
         } while (isOK((char) c));

         if (charsRead == matchString.length()) {
            return fToken;
         } else {
            rewind(scanner);
            return Token.UNDEFINED;
         }

      }

      scanner.unread();
      return Token.UNDEFINED;
   }
 

import org.eclipse.jface.text.rules.Token; //導入依賴的package包/類
private SqlCommentPartitionScanner() {
	IToken commentToken = new Token(SQL_COMMENT);

	List<IPredicateRule> rules= new ArrayList<>();

	rules.add(new EndOfLineRule("--", commentToken, '\\'));
	rules.add(new MultiLineRule("/*", "*/", commentToken, '\\'));
	
	rules.add(new WordPredicateRule(commentToken));
	rules.add(new SingleLineRule("\"", "\"", Token.UNDEFINED, '\\')); 
	rules.add(new SingleLineRule("'", "'", Token.UNDEFINED, '\\'));

	IPredicateRule[] result= new IPredicateRule[rules.size()];
	rules.toArray(result);
	setPredicateRules(result);
}
 

import org.eclipse.jface.text.rules.Token; //導入依賴的package包/類
/**
 * Does the actual evaluation of the stream.
 * 
 * @param scanner The scanner
 * @param count The initial count of {
 * @return <code>fToken</code> on success, <code>Token.UNDEFINED</code> if
 * the match doesn't succeed
 */
private IToken doEvaluate(ICharacterScanner scanner, int count) {
    boolean inString = false;
    int c = scanner.read();

    if (((char) c) == '{') {
        do {
            c = scanner.read();
            if (((char) c) == '{' && !inString)
                count++;
            else if (((char) c) == '}' && !inString)
                count--;
            else if (((char) c) == '"' && !inString)
                inString = true;
            else if (((char) c) == '"' && inString)
                inString = false;
            else if (c == ICharacterScanner.EOF)
            	return Token.UNDEFINED;
        } while (count > 0);
        return fToken;
    }
    scanner.unread();
    return Token.UNDEFINED;
}
 

import org.eclipse.jface.text.rules.Token; //導入依賴的package包/類
public IToken evaluate(ICharacterScanner scanner, boolean resume) {
    if (resume) {
        boolean inString = false;
        do {
            int c = scanner.read();
            if (((char) c) == ',' && !inString)
                break;
            else if (((char) c) == '@') {
                scanner.unread();
                return Token.UNDEFINED;
            } else if (((char) c) == '"' && !inString)
                inString = true;
            else if (((char) c) == '"' && inString)
                inString = false;
            else if (c == ICharacterScanner.EOF)
                return Token.UNDEFINED;
        } while (true);
    }
    return doEvaluate(scanner, 1);
}