Java TokenHierarchy类代码示例

import org.netbeans.api.lexer.TokenHierarchy; //导入依赖的package包/类
public void checkInitialFold() {
    try {
        TokenHierarchy<?> th = info.getTokenHierarchy();
        TokenSequence<JavaTokenId>  ts = th.tokenSequence(JavaTokenId.language());

        while (ts.moveNext()) {
            if (ts.offset() >= initialCommentStopPos)
                break;

            Token<JavaTokenId> token = ts.token();

            if (token.id() == JavaTokenId.BLOCK_COMMENT || token.id() == JavaTokenId.JAVADOC_COMMENT) {
                int startOffset = ts.offset();
                addFold(creator.createInitialCommentFold(startOffset, startOffset + token.length()), startOffset);
                break;
            }
        }
    } catch (ConcurrentModificationException e) {
        //from TokenSequence, document probably changed, stop
        stopped = true;
    }
}
 

import org.netbeans.api.lexer.TokenHierarchy; //导入依赖的package包/类
public void testDocument() throws Exception {
    String text = readJComponentFile();
    // Possibly some extra prepare runs
    int prepareTokenCount = 0;
    for (int i = 0; i < PREPARE_COUNT; i++) { prepareTokenCount += prepareTestDocument(text); }

    // Create the document and token hierarchy again and measure time
    ModificationTextDocument doc = new ModificationTextDocument();
    doc.insertString(0, text, null);
    doc.putProperty(Language.class, JavaTokenId.language());
    TokenHierarchy<?> hi = TokenHierarchy.get(doc);
    TokenSequence<?> ts = hi.tokenSequence();
    long tm = System.currentTimeMillis();
    int tokenCount = 0;
    // Force all the tokens to be initialized
    while (ts.moveNext()) { tokenCount++; }
    tm = System.currentTimeMillis() - tm;
    System.err.println("TH over Swing Document: " + tokenCount + " tokens created in " + tm
            + " ms over document with " + doc.getLength() + " chars; prepareTokenCount="
            + prepareTokenCount + ".");
}
 

import org.netbeans.api.lexer.TokenHierarchy; //导入依赖的package包/类
private synchronized ElementsParserCache getElementsParserCache() {
    if (elementsParserCache == null) {
        CharSequence sourceCode = source.getSourceCode();
        Snapshot snapshot = source.getSnapshot();
        TokenHierarchy hi;
        if (snapshot != null) {
            //use the snapshot's token hierarchy (cached) if possible
            hi = snapshot.getTokenHierarchy();
        } else {
            hi = TokenHierarchy.create(sourceCode, HTMLTokenId.language());
        }
        TokenSequence<HTMLTokenId> tokenSequence = hi.tokenSequence(HTMLTokenId.language());
        elementsParserCache = new ElementsParserCache(source.getSourceCode(), tokenSequence);
    }

    return elementsParserCache;
}
 

import org.netbeans.api.lexer.TokenHierarchy; //导入依赖的package包/类
public void testBasic() {
    String code = "<div></div>";
    //             012345678901
    TokenHierarchy th = TokenHierarchy.create(code, HTMLTokenId.language());
    TokenSequence ts = th.tokenSequence(HTMLTokenId.language());

    ElementsParser parser = ElementsParser.forTokenIndex(code, ts, 0);
    assertTrue(parser.hasNext());

    Element e = parser.next();
    assertNotNull(e);

    assertEquals(ElementType.OPEN_TAG, e.type());
    assertEquals(0, e.from());
    assertEquals(5, e.to());

    assertTrue(parser.hasNext());

    e = parser.next();
    assertNotNull(e);

    assertEquals(ElementType.CLOSE_TAG, e.type());
    assertEquals(5, e.from());
    assertEquals(11, e.to());

}
 

import org.netbeans.api.lexer.TokenHierarchy; //导入依赖的package包/类
public void testCacheBlock() {
    CharSequence code = "<div></div>";
    //             012345678901
    TokenHierarchy th = TokenHierarchy.create(code, HTMLTokenId.language());
    TokenSequence ts = th.tokenSequence(HTMLTokenId.language());

    ElementsParserCache parserCache = new ElementsParserCache(code, ts);

    Iterator<Element> itr = parserCache.createElementsIterator();
    assertNotNull(itr);

    assertTrue(itr.hasNext());

    //now the first block should be loaded
    List<ElementsParserCache.CacheBlock> blocks = parserCache.cacheBlocks;
    assert (blocks.size() == 1);

    ElementsParserCache.CacheBlock block = blocks.get(0);

    assertEquals(0, block.getStartOffset());
    assertEquals(11, block.getEndOffset());

    assertEquals(0, block.getStartIndex());
    assertEquals(2, block.getEndIndex());

}
 

import org.netbeans.api.lexer.TokenHierarchy; //导入依赖的package包/类
/**
 * Checks to see if this document declares any DOCTYPE or not?
 * If exists, it must appear before the first xml tag.
 * @return true if found, else false.
 */
public static boolean isDTDBasedDocument(Document document) {
    ((AbstractDocument)document).readLock();
    try {
        TokenHierarchy th = TokenHierarchy.get(document);
        TokenSequence ts = th.tokenSequence();
        while(ts.moveNext()) {
            Token token = ts.token();
            //if an xml tag is found, we have come too far.
            if(token.id() == XMLTokenId.TAG)
                return false;
            if(token.id() == XMLTokenId.DECLARATION)
                return true;
        }
    } finally {
        ((AbstractDocument)document).readUnlock();
    }
    return false;
}
 

import org.netbeans.api.lexer.TokenHierarchy; //导入依赖的package包/类
private static MimePath getFullMimePath(Document document, int offset) {
    String langPath = null;

    if (document instanceof AbstractDocument) {
        AbstractDocument adoc = (AbstractDocument)document;
        adoc.readLock();
        try {
            List<TokenSequence<?>> list = TokenHierarchy.get(document).embeddedTokenSequences(offset, true);
            if (list.size() > 1) {
                langPath = list.get(list.size() - 1).languagePath().mimePath();
            }
        } finally {
            adoc.readUnlock();
        }
    }

    if (langPath == null) {
        langPath = NbEditorUtilities.getMimeType(document);
    }

    if (langPath != null) {
        return MimePath.parse(langPath);
    } else {
        return null;
    }
}
 

import org.netbeans.api.lexer.TokenHierarchy; //导入依赖的package包/类
private static void assertTokensEqual(String message,
TokenSequence<?> ts, TokenHierarchy tokenHierarchy,
TokenSequence<?> ts2, TokenHierarchy tokenHierarchy2, boolean testLookaheadAndState) {
    Token<?> t = ts.token();
    Token<?> t2 = ts2.token();

    message = messagePrefix(message);
    TestCase.assertEquals(message + "Invalid token id", t.id(), t2.id());
    assertTextEquals(message + "Invalid token text", t.text(), t2.text());
    
    assertTokenOffsetsEqual(message, t.offset(tokenHierarchy), t2.offset(tokenHierarchy2));
    TestCase.assertEquals(message + "Invalid tokenSequence offset", ts.offset(), ts2.offset());

    // Checking LOOKAHEAD and STATE matching in case they are filled in (during tests)
    if (TokenList.LOG.isLoggable(Level.FINE) && testLookaheadAndState) {
        TestCase.assertEquals(message + "Invalid token.lookahead()", lookahead(ts), lookahead(ts2));
        TestCase.assertEquals(message + "Invalid token.state()", state(ts), state(ts2));
    }
    TestCase.assertEquals(message + "Invalid token length", t.length(), t2.length());
    TestCase.assertEquals(message + "Invalid token part", t.partType(), t2.partType());
}
 

import org.netbeans.api.lexer.TokenHierarchy; //导入依赖的package包/类
public void testKnownMimeType() {
    PlainDocument doc = new PlainDocument();
    doc.putProperty("mimeType", MIME_TYPE_KNOWN);
    
    TokenHierarchy th = TokenHierarchy.get(doc);
    ((AbstractDocument)doc).readLock();
    try {
        Language lang = th.tokenSequence().language();
        assertNotNull("There should be language for " + MIME_TYPE_KNOWN, lang);

        assertNotNull("Invalid mime type", lang.mimeType());
        assertEquals("Wrong language's mime type", MIME_TYPE_KNOWN, lang.mimeType());
    } finally {
        ((AbstractDocument)doc).readUnlock();
    }
}
 

import org.netbeans.api.lexer.TokenHierarchy; //导入依赖的package包/类
public void testCacheRefreshedE() {
    TokenHierarchy th = TokenHierarchy.create("abc", TestPlainTokenId.language());
    TokenSequence tokens = th.tokenSequence();
    tokens.moveStart();
    assertEquals(true, tokens.moveNext());
    
    TokenSequence embeddedA = tokens.embedded();
    assertNotNull("There should be an embedded language", embeddedA);
    
    SimpleLanguageProvider.fireTokenLanguageChange();
    
    TokenSequence embeddedB = tokens.embedded();
    assertNotNull("There should be an embedded language", embeddedB);
    
    assertNotSame("The token language cache has not been refreshed", embeddedA, embeddedB);
}
 

import org.netbeans.api.lexer.TokenHierarchy; //导入依赖的package包/类
@Test
public void testLexerSanity() {
    String code = "'Use of assert'://\n" +
                  "   assert /**/ $1 : $2; :: //\n $1 instanceof boolean && $2 instanceof java.lang.Object\n" +
                  "=> if (!$1) throw new /**/ IllegalStateException($2);\n" +
                  ";;//\n";

    for (int cntr = 0; cntr < code.length(); cntr++) {
        String currentpath = code.substring(0, cntr);
        TokenHierarchy<?> hi = TokenHierarchy.create(currentpath, language());
        TokenSequence<?> ts = hi.tokenSequence();

        while (ts.moveNext())
            ;
    }
}
 

import org.netbeans.api.lexer.TokenHierarchy; //导入依赖的package包/类
private static Token<JavaTokenId> findIdentifierSpanImpl(CompilationInfo info, IdentifierTree tree, CompilationUnitTree cu, SourcePositions positions) {
    int start = (int)positions.getStartPosition(cu, tree);
    int endPosition = (int)positions.getEndPosition(cu, tree);
    
    if (start == (-1) || endPosition == (-1))
        return null;

    TokenHierarchy<?> th = info.getTokenHierarchy();
    TokenSequence<JavaTokenId> ts = th.tokenSequence(JavaTokenId.language());

    if (ts.move(start) == Integer.MAX_VALUE) {
        return null;
    }

    if (ts.moveNext()) {
        if (ts.offset() >= start) {
            Token<JavaTokenId> t = ts.token();
            return t;
        }
    }
    
    return null;
}
 

import org.netbeans.api.lexer.TokenHierarchy; //导入依赖的package包/类
@Test
public void testOptions() {
    String text = "<!option1=\"value1,value2\",option2=true>'test': 1 + 1 <!option1=\"value1,value2\",option2=true>=> 1 + 1;;";
    TokenHierarchy<?> hi = TokenHierarchy.create(text, language());
    TokenSequence<?> ts = hi.tokenSequence();
    assertNextTokenEquals(ts, OPTIONS, "<!option1=\"value1,value2\",option2=true>");
    assertNextTokenEquals(ts, CHAR_LITERAL, "'test'");
    assertNextTokenEquals(ts, COLON, ":");
    assertNextTokenEquals(ts, JAVA_SNIPPET, " 1 + 1 ");
    assertNextTokenEquals(ts, OPTIONS, "<!option1=\"value1,value2\",option2=true>");
    assertNextTokenEquals(ts, LEADS_TO, "=>");
    assertNextTokenEquals(ts, JAVA_SNIPPET, " 1 + 1");
    assertNextTokenEquals(ts, DOUBLE_SEMICOLON, ";;");

    assertFalse(ts.moveNext());
}
 

import org.netbeans.api.lexer.TokenHierarchy; //导入依赖的package包/类
private static Token<JavaTokenId> findTokenWithText(CompilationInfo info, String text, int start, int end) {
    TokenHierarchy<?> th = info.getTokenHierarchy();
    TokenSequence<JavaTokenId> ts = th.tokenSequence(JavaTokenId.language()).subSequence(start, end);
    
    while (ts.moveNext()) {
        Token<JavaTokenId> t = ts.token();
        
        if (t.id() == JavaTokenId.IDENTIFIER) {
            boolean nameMatches;
            
            if (!(nameMatches = text.equals(info.getTreeUtilities().decodeIdentifier(t.text()).toString()))) {
                ExpressionTree expr = info.getTreeUtilities().parseExpression(t.text().toString(), new SourcePositions[1]);
                
                nameMatches = expr.getKind() == Kind.IDENTIFIER && text.contentEquals(((IdentifierTree) expr).getName());
            }
            
            if (nameMatches) {
                return t;
            }
        }
    }
    
    return null;
}
 

import org.netbeans.api.lexer.TokenHierarchy; //导入依赖的package包/类
public void test206116WarnAboutRemovingNonVoidExpression() throws Exception {
    prepareTest("test/Test.java", "");
    
    FileObject hint = sourceRoot.createData("test.hint");
    String code = "Math.min(0, 1) => ;;";
    
    TestUtilities.copyStringToFile(hint, code);
    
    TokenHierarchy<?> h = TokenHierarchy.create(code, DeclarativeHintTokenId.language());
    DeclarativeHintsParser.Result res = new DeclarativeHintsParser().parse(hint, code, h.tokenSequence(DeclarativeHintTokenId.language()));
    List<ErrorDescription> errorInstances = HintsTask.computeErrors(res, code, hint);
    List<String> errors = new ArrayList<String>();
    
    for (ErrorDescription ed : errorInstances) {
        errors.add(ed.toString());
    }
    
    assertEquals(Arrays.asList("0:0-0:15:warning:ERR_RemoveExpression"), errors);
}