Java Lexer类代码示例

import com.intellij.lexer.Lexer; //导入依赖的package包/类
@Override
public MergeFunction getMergeFunction() {
  return ((final IElementType type, final Lexer originalLexer) -> {
    if (type == SoyTypes.OTHER || type == TokenType.WHITE_SPACE) {
      IElementType returnType =  type;
      while (originalLexer.getTokenType() == SoyTypes.OTHER
          || originalLexer.getTokenType() == TokenType.WHITE_SPACE) {
        if (originalLexer.getTokenType() == SoyTypes.OTHER) {
          returnType = SoyTypes.OTHER;
        }
        originalLexer.advance();
      }
      return returnType;
    }

    return type;
  });
}
 

import com.intellij.lexer.Lexer; //导入依赖的package包/类
public static void addWordHonoringEscapeSequences(CharSequence editorText,
                                                  TextRange literalTextRange,
                                                  int cursorOffset,
                                                  Lexer lexer,
                                                  List<TextRange> result) {
  lexer.start(editorText, literalTextRange.getStartOffset(), literalTextRange.getEndOffset());

  while (lexer.getTokenType() != null) {
    if (lexer.getTokenStart() <= cursorOffset && cursorOffset < lexer.getTokenEnd()) {
      if (StringEscapesTokenTypes.STRING_LITERAL_ESCAPES.contains(lexer.getTokenType())) {
        result.add(new TextRange(lexer.getTokenStart(), lexer.getTokenEnd()));
      }
      else {
        TextRange word = getWordSelectionRange(editorText, cursorOffset, JAVA_IDENTIFIER_PART_CONDITION);
        if (word != null) {
          result.add(new TextRange(Math.max(word.getStartOffset(), lexer.getTokenStart()),
                                   Math.min(word.getEndOffset(), lexer.getTokenEnd())));
        }
      }
      break;
    }
    lexer.advance();
  }
}
 

import com.intellij.lexer.Lexer; //导入依赖的package包/类
protected void checkCorrectRestart(String text) {
  Lexer mainLexer = createLexer();
  String allTokens = printTokens(text, 0, mainLexer);

  Lexer auxLexer = createLexer();
  auxLexer.start(text);
  while (true) {
    IElementType type = auxLexer.getTokenType();
    if (type == null) {
      break;
    }
    if (auxLexer.getState() == 0) {
      int tokenStart = auxLexer.getTokenStart();
      String subTokens = printTokens(text, tokenStart, mainLexer);
      if (!allTokens.endsWith(subTokens)) {
        assertEquals("Restarting impossible from offset " + tokenStart + "; lexer state should not return 0 at this point", allTokens, subTokens);
      }
    }
    auxLexer.advance();
  }
}
 

import com.intellij.lexer.Lexer; //导入依赖的package包/类
@Override
public MergeFunction getMergeFunction() {
    return new MergeFunction() {

        @Override
        public IElementType merge(IElementType type, Lexer originalLexer) {
            if (!allMergables.contains(type)) {
                return type;
            }

            TokenSet merging = tokensToMerge.contains(type) ? tokensToMerge : tokensToMerge2;

            while (true) {
                final IElementType tokenType = originalLexer.getTokenType();
                if (!merging.contains(tokenType)) break;
                originalLexer.advance();
            }

            return merging == tokensToMerge ? LONGSTRING : LONGCOMMENT;
        }
    };
}
 

import com.intellij.lexer.Lexer; //导入依赖的package包/类
private ASTNode createAST(String text) {
  ParserDefinition definition = new BuildParserDefinition();
  PsiParser parser = definition.createParser(getProject());
  Lexer lexer = definition.createLexer(getProject());
  PsiBuilderImpl psiBuilder =
      new PsiBuilderImpl(
          getProject(), null, definition, lexer, new CharTableImpl(), text, null, null);
  PsiBuilderAdapter adapter =
      new PsiBuilderAdapter(psiBuilder) {
        @Override
        public void error(String messageText) {
          super.error(messageText);
          errors.add(messageText);
        }
      };
  return parser.parse(definition.getFileNodeType(), adapter);
}
 

import com.intellij.lexer.Lexer; //导入依赖的package包/类
@Nullable
public static String parseQName(Lexer lexer) {
  String name;
  if (lexer.getTokenType() == XPathTokenTypes.NCNAME) {
    name = lexer.getTokenText();
    lexer.advance();
    if (lexer.getTokenType() == XPathTokenTypes.COL) {
      lexer.advance();
      assert lexer.getTokenType() == XPathTokenTypes.NCNAME;
      name += (":" + lexer.getTokenText());
      lexer.advance();
    }
  } else {
    name = null;
  }
  return name;
}
 

import com.intellij.lexer.Lexer; //导入依赖的package包/类
protected void registerHandler(IElementType elementType, TokenHandler value) {
  final TokenHandler tokenHandler = tokenHandlers.get(elementType);

  if (tokenHandler != null) {
    final TokenHandler newHandler = value;
    value = new TokenHandler() {
      @Override
      public void handleElement(final Lexer lexer) {
        tokenHandler.handleElement(lexer);
        newHandler.handleElement(lexer);
      }
    };
  }

  tokenHandlers.put(elementType,value);
}
 

import com.intellij.lexer.Lexer; //导入依赖的package包/类
private static void doTest(String text, String[] expectedTokens,Lexer lexer) {
  lexer.start(text);
  int idx = 0;
  while (lexer.getTokenType() != null) {
    if (idx >= expectedTokens.length) fail("Too many tokens");
    String tokenName = lexer.getTokenType().toString();
    String expectedTokenType = expectedTokens[idx++];
    String expectedTokenText = expectedTokens[idx++];
    assertEquals(expectedTokenType, tokenName);
    String tokenText = lexer.getBufferSequence().subSequence(lexer.getTokenStart(), lexer.getTokenEnd()).toString();
    assertEquals(expectedTokenText, tokenText);
    lexer.advance();
  }

  if (idx < expectedTokens.length) fail("Not enough tokens");
}
 

import com.intellij.lexer.Lexer; //导入依赖的package包/类
@Test
public void testError2() {
    Lexer l = createLexer();

    startSkippingHeaders(l, "a: x\n" +
            "b: y\n" +
            "ab");

    assertAndAdvance(l, YamlTokenTypes.YAML_Key);
    assertAndAdvance(l, YamlTokenTypes.YAML_Value);
    assertAndAdvance(l, YamlTokenTypes.YAML_Whitespace);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 3, 5, "x\n");

    assertAndAdvance(l, YamlTokenTypes.YAML_Key);
    assertAndAdvance(l, YamlTokenTypes.YAML_Value);
    assertAndAdvance(l, YamlTokenTypes.YAML_Whitespace);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 8, 10, "y\n");

    // Simplified lexer ignores this error
    //assertAndAdvance(l, YamlTokenTypes.YAML_Error);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar);

    assertEquals(null, l.getTokenType());
}
 

import com.intellij.lexer.Lexer; //导入依赖的package包/类
@Test
public void testError3() {
    Lexer l = createLexer();

    startSkippingHeaders(l, "a: x\n" +
            "b: y\n" +
            "@bad!");

    assertAndAdvance(l, YamlTokenTypes.YAML_Key);
    assertAndAdvance(l, YamlTokenTypes.YAML_Value);
    assertAndAdvance(l, YamlTokenTypes.YAML_Whitespace);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 3, 5, "x\n");

    assertAndAdvance(l, YamlTokenTypes.YAML_Key);
    assertAndAdvance(l, YamlTokenTypes.YAML_Value);
    assertAndAdvance(l, YamlTokenTypes.YAML_Whitespace);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 8, 10, "y\n");

    assertAndAdvance(l, YamlTokenTypes.YAML_Error);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar);

    assertEquals(null, l.getTokenType());
}
 

import com.intellij.lexer.Lexer; //导入依赖的package包/类
@Override
public int getErrorsCount(final CharSequence seq, Language fileLanguage, final Project project) {
  Lexer lexer = JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);

  lexer.start(seq);
  if (lexer.getTokenType() != JavaTokenType.LBRACE) return IErrorCounterReparseableElementType.FATAL_ERROR;
  lexer.advance();
  int balance = 1;
  while (true) {
    IElementType type = lexer.getTokenType();
    if (type == null) break;
    if (balance == 0) return IErrorCounterReparseableElementType.FATAL_ERROR;
    if (type == JavaTokenType.LBRACE) {
      balance++;
    }
    else if (type == JavaTokenType.RBRACE) {
      balance--;
    }
    lexer.advance();
  }
  return balance;
}
 

import com.intellij.lexer.Lexer; //导入依赖的package包/类
private static boolean canStickJavaTokens(ASTNode token1, ASTNode token2) {
  IElementType type1 = token1.getElementType();
  IElementType type2 = token2.getElementType();

  Pair<IElementType, IElementType> pair = Pair.create(type1, type2);
  Boolean res = myCanStickJavaTokensMatrix.get(pair);
  if (res == null) {
    if (!checkToken(token1) || !checkToken(token2)) return true;
    String text = token1.getText() + token2.getText();
    Lexer lexer = JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);
    lexer.start(text);
    boolean canMerge = lexer.getTokenType() == type1;
    lexer.advance();
    canMerge &= lexer.getTokenType() == type2;
    res = canMerge;
    myCanStickJavaTokensMatrix.put(pair, res);
  }
  return res.booleanValue();
}
 

import com.intellij.lexer.Lexer; //导入依赖的package包/类
protected PsiElement createEmptyPsiField() {
    final String text = "<undefined>";
    final IElementType type = CsvTypes.FIELD;
    final Project project = this.element.getProject();
    final PsiManager psiManager = PsiManager.getInstance(project);
    final DummyHolder dummyHolder = DummyHolderFactory.createHolder(psiManager, null);
    final FileElement fileElement = dummyHolder.getTreeElement();
    final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(CsvLanguage.INSTANCE);
    final Lexer lexer = parserDefinition.createLexer(project);
    final PsiBuilder psiBuilder = PsiBuilderFactory.getInstance().createBuilder(project, fileElement, lexer, CsvLanguage.INSTANCE, text);
    final ASTNode node = parserDefinition.createParser(project).parse(type, psiBuilder);
    fileElement.rawAddChildren((com.intellij.psi.impl.source.tree.TreeElement)node);
    return node.getPsi();
}
 

import com.intellij.lexer.Lexer; //导入依赖的package包/类
@Override
@NotNull
public Lexer getHighlightingLexer() {
  if (myIsDtd) {
    return new DtdLexer(true);
  } else if (myIsXHtml) {
    return new XHtmlHighlightingLexer();
  } else {
    return new XmlHighlightingLexer();
  }
}
 

import com.intellij.lexer.Lexer; //导入依赖的package包/类
private boolean replaceTokenNotFollowedBy(Lexer baseLexer, final IElementType replacement, IElementType nextToken) {
  if (!lookingAt(baseLexer, nextToken)) {
    treatAs(baseLexer, replacement);
    return true;
  }
  return false;
}