Java TokenSequence.embedded方法代码示例

import org.netbeans.api.lexer.TokenSequence; //导入方法依赖的package包/类
/**
 * @since 0.21
 */
public static TokenSequence<JavaTokenId> getJavaTokenSequence(final TokenHierarchy hierarchy, final int offset) {
    if (hierarchy != null) {
        TokenSequence<?> ts = hierarchy.tokenSequence();
        while(ts != null && (offset == 0 || ts.moveNext())) {
            ts.move(offset);
            if (ts.language() == JavaTokenId.language()) {
                return (TokenSequence<JavaTokenId>)ts;
            }
            if (!ts.moveNext() && !ts.movePrevious()) {
                return null;
            }
            ts = ts.embedded();
        }
    }
    return null;
}
 

import org.netbeans.api.lexer.TokenSequence; //导入方法依赖的package包/类
private static void dumpSequence(TokenSequence<?> seq, StringBuilder sb) {
    if (seq == null) {
        sb.append("Inactive TokenHierarchy"); //NOI18N
    } else {
        for(seq.moveStart(); seq.moveNext(); ) {
            TokenSequence<?> emSeq = seq.embedded();
            if (emSeq != null) {
                dumpSequence(emSeq, sb);
            } else {
                Token<?> token = seq.token();
                sb.append("<"); //NOI18N
                sb.append(String.format("%3s", seq.offset())).append(", "); //NOI18N
                sb.append(String.format("%3s", seq.offset() + token.length())).append(", "); //NOI18N
                sb.append(String.format("%+3d", token.length())).append("> : "); //NOI18N
                sb.append(tokenId(token.id(), true)).append(" : '"); //NOI18N
                sb.append(tokenText(token));
                sb.append("'\n"); //NOI18N
            }
        }
    }
}
 

import org.netbeans.api.lexer.TokenSequence; //导入方法依赖的package包/类
private boolean testCreateAnnotation(TokenHierarchy hi, TokenSequence ts, ASTItem item, LanguagesAnnotation la) throws BadLocationException {
    if (ts.language () == null)
        throw new NullPointerException ("ts.language()==null");
    if (ts.language ().mimeType () == null)
        throw new NullPointerException ("TokenSequence.mimeType==null");
    if (ts.language().mimeType().equals(item.getMimeType())) {
            Token t = ts.token();
            if (t == null) throw new NullPointerException ();
            Position position = doc.createPosition(t.offset(hi));
            la.setPosition (position);
            doc.addAnnotation(position, t.length(), la);
            return true;
        } else {
            ts = ts.embedded();
            if(ts == null) {
                return false;
            } else {
                return ts.moveNext() ? testCreateAnnotation(hi, ts, item, la) : false;
            }
        }
}
 

import org.netbeans.api.lexer.TokenSequence; //导入方法依赖的package包/类
private static TokenSequence getDeepestTokenSequence (
    TokenHierarchy  tokenHierarchy,
    int             offset
) {
    TokenSequence tokenSequence = tokenHierarchy.tokenSequence ();
    while(tokenSequence != null) {
        tokenSequence.move(offset - 1);
        if(!tokenSequence.moveNext()) {
            break;
        }
        TokenSequence ts = tokenSequence.embedded();
        if(ts == null) {
            return tokenSequence;
        } else {
            tokenSequence = ts;
        }
    }
    return tokenSequence;
}
 

import org.netbeans.api.lexer.TokenSequence; //导入方法依赖的package包/类
public void testShortDoc() throws Exception {
    //             000000000011111111112222222222
    //             012345678901234567890123456789
    String text = "a<b>c";
    ModificationTextDocument doc = new ModificationTextDocument();
    doc.insertString(0, text, null);
    doc.putProperty(Language.class, TestJoinTopTokenId.language());
    TokenHierarchy<?> hi = TokenHierarchy.get(doc);
    ((AbstractDocument)doc).readLock();
    try {
        TokenSequence<?> ts = hi.tokenSequence();
        assertTrue(ts.moveNext());
        ts.embedded(); // Creates JTL
    } finally {
        ((AbstractDocument)doc).readUnlock();
    }
}
 

import org.netbeans.api.lexer.TokenSequence; //导入方法依赖的package包/类
public static boolean isJavadocContext(TokenHierarchy hierarchy, int offset) {
    TokenSequence<JavaTokenId> ts = SourceUtils.getJavaTokenSequence(hierarchy, offset);
    if (!movedToJavadocToken(ts, offset)) {
        return false;
    }
    
    TokenSequence<JavadocTokenId> jdts = ts.embedded(JavadocTokenId.language());
    if (jdts == null) {
        return false;
    } else if (jdts.isEmpty()) {
        return isEmptyJavadoc(ts.token(), offset - ts.offset());
    }
    
    jdts.move(offset);
    if (!jdts.moveNext() && !jdts.movePrevious()) {
        return false;
    }
    
    // this checks /** and */ headers
    return isInsideToken(jdts, offset) && !isInsideIndent(jdts.token(), offset - jdts.offset());
}
 

import org.netbeans.api.lexer.TokenSequence; //导入方法依赖的package包/类
public static void incCheckNested(String message, Document doc,
        TokenSequence<?> batch, TokenHierarchy<?> batchTH,
        TokenSequence<?> inc, TokenHierarchy<?> incTH
) {
    int i = 0;
    while (inc.moveNext()) {
        TestCase.assertTrue("No more tokens in batch token sequence", batch.moveNext());
        TokenSequence<?> batchE = batch.embedded();
        TokenSequence<?> incE = inc.embedded();
        String messageE = message + "->[" + i + "]";
        if (incE != null) {
            TestCase.assertNotNull("Batch embedded sequence is null", batchE);
            assertTokenSequencesEqual(messageE, batchE, batchTH, incE, incTH, true, true);
            incE.moveStart();
            batchE.moveStart();

            incCheckNested(messageE, doc, batchE, batchTH, incE, incTH);
        } else { // Inc embedded is null
            TestCase.assertNull("Batch embedded sequence non-null", batchE);
        }
        i++;
    }
}
 

import org.netbeans.api.lexer.TokenSequence; //导入方法依赖的package包/类
/**Find span of the name in the DocTree's reference tree (see {@link #getReferenceName(com.sun.source.util.DocTreePath)}
 * identifier in the source. Returns starting and ending offset of the name in
 * the source code that was parsed (ie. {@link CompilationInfo.getText()}, which
 * may differ from the positions in the source document if it has been already
 * altered.
 * 
 * @param ref reference for which the identifier should be found
 * @return the span of the name, or null if cannot be found
 * @since 0.124
 */
public int[] findNameSpan(DocCommentTree docTree, ReferenceTree ref) {
    Name name = ((DCReference) ref).memberName;
    if (name == null || !SourceVersion.isIdentifier(name)) {
        //names like "<error>", etc.
        return null;
    }
    
    int pos = (int) info.getDocTrees().getSourcePositions().getStartPosition(info.getCompilationUnit(), docTree, ref);
    
    if (pos < 0)
        return null;
    
    TokenSequence<JavaTokenId> tokenSequence = info.getTokenHierarchy().tokenSequence(JavaTokenId.language());
    
    tokenSequence.move(pos);
    
    if (!tokenSequence.moveNext() || tokenSequence.token().id() != JavaTokenId.JAVADOC_COMMENT) return null;
    
    TokenSequence<JavadocTokenId> jdocTS = tokenSequence.embedded(JavadocTokenId.language());
    
    jdocTS.move(pos);
    
    boolean wasNext;
    
    while ((wasNext = jdocTS.moveNext()) && jdocTS.token().id() != JavadocTokenId.HASH)
        ;
    
    if (wasNext && jdocTS.moveNext()) {
        if (jdocTS.token().id() == JavadocTokenId.IDENT &&
            name.contentEquals(jdocTS.token().text())) {
            return new int[] {
                jdocTS.offset(),
                jdocTS.offset() + jdocTS.token().length()
            };
        }
    }
    
    return null;
}
 

import org.netbeans.api.lexer.TokenSequence; //导入方法依赖的package包/类
public void testFindIndentTaskFactory() throws BadLocationException {
    Document doc = new PlainDocument();
    doc.insertString(0, "first line\nsecond-line", null);
    doc.putProperty("mimeType", TestLineTokenId.MIME_TYPE);
    
    TokenHierarchy hi = TokenHierarchy.get(doc);
    assertNotNull(hi);
    TokenSequence<?> ts = hi.tokenSequence();
    assertTrue(ts.moveNext());
    TokenSequence<?> ets = ts.embedded();
    assertNotNull(ets);
    // There should be two language paths - root one and one with plain embedding
    assertEquals(2, hi.languagePaths().size());
    
    Reformat reformat = Reformat.get(doc);
    reformat.lock();
    try {
        //doc.atomicLock();
        try {
            reformat.reformat(0, doc.getLength());
        } finally {
            //doc.atomicUnlock();
        }
    } finally {
        reformat.unlock();
    }
    
    String text = doc.getText(0, doc.getLength());
    // lineReformatTaskFactory should be called first and its mimetype at the begining of document
    // and plainReformatTaskFactory should be called second and also add its mimetype at the begining of document
    assertEquals(TestPlainTokenId.MIME_TYPE + "/" + TestLineTokenId.MIME_TYPE + "/first line\nsecond-line", text);
}
 

import org.netbeans.api.lexer.TokenSequence; //导入方法依赖的package包/类
public void testLanguagesEmbeddingMapMT() throws Exception {
    Document doc = new PlainDocument();
    doc.putProperty("mimeType", "text/x-simple-plain");
    // All words have to be longer than 3 characters
    doc.insertString(0, "Hello 1234 0xFF00", SimpleAttributeSet.EMPTY);
    
    TokenHierarchy th = TokenHierarchy.get(doc);
    assertNotNull("Can't find token hierarchy for a text/x-simple-plain document", th);
    
    TokenSequence seq = th.tokenSequence();
    Language lang = seq.language();
    assertNotNull("Can't find language for text/x-simple-plain", lang);
    assertEquals("Wrong language", "text/x-simple-plain", lang.mimeType());
    
    for(int i = 0; i < seq.tokenCount(); i++) {
        seq.moveIndex(i);
        assertTrue(seq.moveNext());
        Token token = seq.token();
        
        if (token.id() == SimplePlainTokenId.WORD) {
            TokenSequence embeddedSeq = seq.embedded();
            assertNotNull("Can't find embedded token sequence", embeddedSeq);

            Language embeddedLang = embeddedSeq.language();
            assertNotNull("Can't find language of the embedded sequence", embeddedLang);
            assertEquals("Wrong language of the embedded sequence", "text/x-simple-char", embeddedLang.mimeType());
            
            embeddedSeq.moveStart();
            assertTrue("Embedded sequence has no tokens (moveFirst)", embeddedSeq.moveNext());
            assertEquals("Wrong startSkipLength", 1, embeddedSeq.offset() - seq.offset());
            
            embeddedSeq.moveEnd();
            assertTrue("Embedded sequence has no tokens (moveLast)", embeddedSeq.movePrevious());
            assertEquals("Wrong endSkipLength", 2, 
                (seq.offset() + seq.token().length()) - (embeddedSeq.offset() + embeddedSeq.token().length()));
        }
    }
}
 

import org.netbeans.api.lexer.TokenSequence; //导入方法依赖的package包/类
public void testEmbeddingActivityChange() throws Exception {
    ModificationTextDocument doc = new ModificationTextDocument();
    // Assign a language to the document
    doc.putProperty(Language.class,TestTokenId.language());
    doc.insertString(0, "a/*abc def*/", null);
    LexerTestUtilities.initLastTokenHierarchyEventListening(doc);
    TokenHierarchy<?> hi = TokenHierarchy.get(doc);
    ((AbstractDocument)doc).readLock();
    try {
        TokenSequence<?> ts = hi.tokenSequence();
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "a", 0);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestTokenId.BLOCK_COMMENT, "/*abc def*/", 1);
        TokenSequence<?> ets = ts.embedded();
        assertNotNull(ets);
        assertTrue(ts.moveNext());
        assertTrue(ets.moveNext());
        LexerTestUtilities.assertTokenEquals(ets,TestPlainTokenId.WORD, "abc", 3);
        assertTrue(ets.moveNext());
        LexerTestUtilities.assertTokenEquals(ets,TestPlainTokenId.WHITESPACE, " ", 6);
        assertTrue(ets.moveNext());
        LexerTestUtilities.assertTokenEquals(ets,TestPlainTokenId.WORD, "def", 7);
        assertFalse(ets.moveNext());
    } finally {
        ((AbstractDocument)doc).readUnlock();
    }

    MutableTextInput input = (MutableTextInput) doc.getProperty(MutableTextInput.class);
    final TokenHierarchyControl control = input.tokenHierarchyControl();
    doc.runAtomic(new Runnable() {
        @Override
        public void run() {
            control.setActive(false);
            control.setActive(true);
        }
    });

}
 

import org.netbeans.api.lexer.TokenSequence; //导入方法依赖的package包/类
public static TokenSequence getTokenSequence (Document document, int offset) {
    TokenHierarchy tokenHierarchy = TokenHierarchy.get (document);
    if (tokenHierarchy == null) return null;
    TokenSequence tokenSequence = tokenHierarchy.tokenSequence ();
    if (tokenSequence == null) return null;
    while (true) {
        tokenSequence.move (offset);
        if (!tokenSequence.moveNext ()) return tokenSequence;
        TokenSequence tokenSequence2 = tokenSequence.embedded ();
        if (tokenSequence2 == null) return tokenSequence;
        tokenSequence = tokenSequence2;
    }
}
 

import org.netbeans.api.lexer.TokenSequence; //导入方法依赖的package包/类
private OffsetRange getCoreHtmlReferenceSpan(Document doc, int caretOffset) {
    TokenHierarchy hi = TokenHierarchy.get(doc);
    final TokenSequence<HTMLTokenId> ts = Utils.getJoinedHtmlSequence(hi, caretOffset);
    if (ts == null) {
        return null;
    }

    //tag attribute value hyperlinking
    if (ts.token().id() == HTMLTokenId.VALUE) {
        return new AttributeValueAction<OffsetRange>(hi, ts) {

            @Override
            public OffsetRange resolve() {
                if (tagName != null && attrName != null) {
                    ValueCompletion<HtmlCompletionItem> support = AttrValuesCompletion.getSupport(tagName, attrName);
                    if (AttrValuesCompletion.FILE_NAME_SUPPORT == support) {
                        //some file to hyperlink to
                        return valueRange;
                    }
                }
                return null;
            }
        }.run();

    } else if (ts.token().id() == HTMLTokenId.VALUE_CSS) {
        //css class or id hyperlinking
        TokenSequence<CssTokenId> cssTs = ts.embedded(CssTokenId.language());
        if (cssTs != null) {
            cssTs.move(caretOffset);
            if (cssTs.moveNext() || cssTs.movePrevious()) {
                if (cssTs.token().id() == CssTokenId.IDENT) {
                    return new OffsetRange(cssTs.offset(), cssTs.offset() + cssTs.token().length());
                }
            }
        }
    }

    return null;
}
 

import org.netbeans.api.lexer.TokenSequence; //导入方法依赖的package包/类
public static TokenSequence<JavadocTokenId> findJavadocTokenSequence(CompilationInfo javac, int offset) {
    TokenSequence<JavaTokenId> ts = SourceUtils.getJavaTokenSequence(javac.getTokenHierarchy(), offset);
    if (ts == null || !movedToJavadocToken(ts, offset)) {
        return null;
    }
    
    TokenSequence<JavadocTokenId> jdts = ts.embedded(JavadocTokenId.language());
    if (jdts == null) {
        return null;
    }
    
    jdts.move(offset);
    return jdts;
}
 

import org.netbeans.api.lexer.TokenSequence; //导入方法依赖的package包/类
private void handleValue(Snapshot snapshot, TokenSequence<HTMLTokenId> ts, List<Embedding> embeddings) {
    if (ts.embedded(JS_LANGUAGE) != null) {
        //has javascript embedding
        embeddings.addAll(createEmbedding(snapshot, "(function(){\n")); //NOI18N
        int diff = Utils.isAttributeValueQuoted(ts.token().text()) ? 1 : 0;
        embeddings.addAll(createEmbedding(snapshot, ts.offset() + diff, ts.token().length() - diff * 2));
        embeddings.addAll(createEmbedding(snapshot, ";\n});\n")); //NOI18N
    }
}