当前位置: 首页>>代码示例>>Java>>正文

Java TokenHierarchy.create方法代码示例

本文整理汇总了Java中org.netbeans.api.lexer.TokenHierarchy.create方法的典型用法代码示例。如果您正苦于以下问题：Java TokenHierarchy.create方法的具体用法？Java TokenHierarchy.create怎么用？Java TokenHierarchy.create使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.netbeans.api.lexer.TokenHierarchy的用法示例。


在下文中一共展示了TokenHierarchy.create方法的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: testIdentifiers

import org.netbeans.api.lexer.TokenHierarchy; //导入方法依赖的package包/类
public void testIdentifiers() {
    String text = "a ab aB2 2a x\nyZ\r\nz";
    TokenHierarchy<?> hi = TokenHierarchy.create(text, JavaTokenId.language());
    TokenSequence<?> ts = hi.tokenSequence();
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "a");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "ab");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "aB2");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.INT_LITERAL, "2");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "a");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "x");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, "\n");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "yZ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, "\r\n");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "z");
}
开发者ID:apache，项目名称:incubator-netbeans，代码行数:20，代码来源:JavaLexerBatchTest.java

示例2: testIssue218629

import org.netbeans.api.lexer.TokenHierarchy; //导入方法依赖的package包/类
public void testIssue218629() {
        String text = "<div>שלום עולם</div>";
        //             01234567890123456789
        TokenHierarchy<String> th = TokenHierarchy.create(text, HTMLTokenId.language());
        TokenSequence<HTMLTokenId> ts = th.tokenSequence(HTMLTokenId.language());
        
        StringBuilder b = new StringBuilder();
        ts.moveStart();
        while(ts.moveNext()) {
            Token<HTMLTokenId> t = ts.token();
            b.append('"');
            b.append(t.text());
            b.append('"');
            b.append("\t\t");
            b.append(t.id().name());
            b.append("\t\t");
            b.append(ts.offset());
            b.append('-');
            b.append(ts.offset() + t.length());
            b.append('\n');
        }
        
//        System.out.println(b);
    }
开发者ID:apache，项目名称:incubator-netbeans，代码行数:25，代码来源:HtmlLexerTest.java

示例3: testOperators

import org.netbeans.api.lexer.TokenHierarchy; //导入方法依赖的package包/类
public void testOperators() {
    String text = "^ ^= % %= * *= / /= = ==";
    TokenHierarchy<?> hi = TokenHierarchy.create(text, JavaTokenId.language());
    TokenSequence<?> ts = hi.tokenSequence();
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CARET, "^");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.CARETEQ, "^=");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.PERCENT, "%");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.PERCENTEQ, "%=");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STAR, "*");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STAREQ, "*=");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.SLASH, "/");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.SLASHEQ, "/=");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.EQ, "=");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.WHITESPACE, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.EQEQ, "==");
}
开发者ID:apache，项目名称:incubator-netbeans，代码行数:25，代码来源:JavaLexerBatchTest.java

示例4: testGCedE

import org.netbeans.api.lexer.TokenHierarchy; //导入方法依赖的package包/类
public void testGCedE() {
    TokenHierarchy th = TokenHierarchy.create("abc", TestPlainTokenId.language());
    TokenSequence tokens = th.tokenSequence();
    tokens.moveStart();
    assertEquals(true, tokens.moveNext());
    
    TokenSequence embedded = tokens.embedded();
    assertNotNull("There should be an embedded language", embedded);
    
    WeakReference<Language> refLang = new WeakReference<Language>(embedded.language());
    embedded = null;

    WeakReference<Token> refToken = new WeakReference<Token>(tokens.token());
    tokens = null;
    th = null;
    
    // This no longer works after the language is statically held in the xxTokenId by the new convention
    //assertGC("The embedded language has not been GCed", refLang);
    assertGC("The token with embedded language has not been GCed", refToken);
}
开发者ID:apache，项目名称:incubator-netbeans，代码行数:21，代码来源:LanguageManagerTest.java

示例5: testInterferenceBraceIdent

import org.netbeans.api.lexer.TokenHierarchy; //导入方法依赖的package包/类
public void testInterferenceBraceIdent() {
    String text = "() -> {A::a();}";
    InputAttributes attr = new InputAttributes();
    attr.setValue(JavaTokenId.language(), "version", Integer.valueOf(7), true);
    TokenHierarchy<?> hi = TokenHierarchy.create(text, false, JavaTokenId.language(), EnumSet.of(JavaTokenId.WHITESPACE), attr);
    TokenSequence<?> ts = hi.tokenSequence();

    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.LPAREN, "(");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.RPAREN, ")");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.ARROW, "->");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.LBRACE, "{");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "A");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.COLONCOLON, "::");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "a");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.LPAREN, "(");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.RPAREN, ")");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.SEMICOLON, ";");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.RBRACE, "}");

    assertFalse(ts.moveNext());
}
开发者ID:apache，项目名称:incubator-netbeans，代码行数:22，代码来源:JavaLexerBatchTest.java

示例6: testSimple

import org.netbeans.api.lexer.TokenHierarchy; //导入方法依赖的package包/类
@Test
public void testSimple() {
    String text = " \'test\': 1 + 1 => \'fix\': 1 + 1;;";
    TokenHierarchy<?> hi = TokenHierarchy.create(text, language());
    TokenSequence<?> ts = hi.tokenSequence();
    assertNextTokenEquals(ts, WHITESPACE, " ");
    assertNextTokenEquals(ts, CHAR_LITERAL, "\'test\'");
    assertNextTokenEquals(ts, COLON, ":");
    assertNextTokenEquals(ts, JAVA_SNIPPET, " 1 + 1 ");
    assertNextTokenEquals(ts, LEADS_TO, "=>");
    assertNextTokenEquals(ts, WHITESPACE, " ");
    assertNextTokenEquals(ts, CHAR_LITERAL, "\'fix\'");
    assertNextTokenEquals(ts, COLON, ":");
    assertNextTokenEquals(ts, JAVA_SNIPPET, " 1 + 1");
    assertNextTokenEquals(ts, DOUBLE_SEMICOLON, ";;");

    assertFalse(ts.moveNext());
}
开发者ID:apache，项目名称:incubator-netbeans，代码行数:19，代码来源:DeclarativeHintLexerTest.java

示例7: testNoBinaryLiterals

import org.netbeans.api.lexer.TokenHierarchy; //导入方法依赖的package包/类
public void testNoBinaryLiterals() {
    String text = "0b101 0B101 0b101l 0b101L";
    InputAttributes attr = new InputAttributes();
    attr.setValue(JavaTokenId.language(), "version", Integer.valueOf(5), true);
    TokenHierarchy<?> hi = TokenHierarchy.create(text, false, JavaTokenId.language(), EnumSet.of(JavaTokenId.WHITESPACE), attr);
    TokenSequence<?> ts = hi.tokenSequence();

    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.INT_LITERAL, "0");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "b101");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.INT_LITERAL, "0");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "B101");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.INT_LITERAL, "0");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "b101l");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.INT_LITERAL, "0");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "b101L");
}
开发者ID:apache，项目名称:incubator-netbeans，代码行数:17，代码来源:JavaLexerBatchTest.java

示例8: testMultiple

import org.netbeans.api.lexer.TokenHierarchy; //导入方法依赖的package包/类
@Test
public void testMultiple() {
    String text = "'test': 1 + 1 => 1 + 1;;'test2': 1 + 1 => 1 + 1;;";
    TokenHierarchy<?> hi = TokenHierarchy.create(text, language());
    TokenSequence<?> ts = hi.tokenSequence();
    assertNextTokenEquals(ts, CHAR_LITERAL, "'test'");
    assertNextTokenEquals(ts, COLON, ":");
    assertNextTokenEquals(ts, JAVA_SNIPPET, " 1 + 1 ");
    assertNextTokenEquals(ts, LEADS_TO, "=>");
    assertNextTokenEquals(ts, JAVA_SNIPPET, " 1 + 1");
    assertNextTokenEquals(ts, DOUBLE_SEMICOLON, ";;");
    assertNextTokenEquals(ts, CHAR_LITERAL, "'test2'");
    assertNextTokenEquals(ts, COLON, ":");
    assertNextTokenEquals(ts, JAVA_SNIPPET, " 1 + 1 ");
    assertNextTokenEquals(ts, LEADS_TO, "=>");
    assertNextTokenEquals(ts, JAVA_SNIPPET, " 1 + 1");
    assertNextTokenEquals(ts, DOUBLE_SEMICOLON, ";;");

    assertFalse(ts.moveNext());
}
开发者ID:apache，项目名称:incubator-netbeans，代码行数:21，代码来源:DeclarativeHintLexerTest.java

示例9: testOnlyAtSymbolLexing

import org.netbeans.api.lexer.TokenHierarchy; //导入方法依赖的package包/类
public void testOnlyAtSymbolLexing() throws Exception {
    String input = "@";
    TokenHierarchy th = TokenHierarchy.create(input, CssTokenId.language());
    TokenSequence ts = th.tokenSequence();
    ts.moveStart();

    assertTrue(ts.moveNext());
    Token<CssTokenId> token = ts.token();

    assertNotNull(token);
    assertEquals(CssTokenId.ERROR, token.id());

}
开发者ID:apache，项目名称:incubator-netbeans，代码行数:14，代码来源:NbCss3LexerTest.java

示例10: getTokenHierarchy

import org.netbeans.api.lexer.TokenHierarchy; //导入方法依赖的package包/类
@NonNull
public final TokenHierarchy<?> getTokenHierarchy() throws IOException {
    if (this.tokens == null) {
        final CharBuffer charBuffer = getCharContent(true);
        InputAttributes attrs = new InputAttributes();
        attrs.setValue(JavaTokenId.language(), "fileName", (Supplier<String>) () -> getName(), true); //NOI18N            
        this.tokens = TokenHierarchy.create(charBuffer, false, JavaTokenId.language(), null, attrs); //TODO: .createSnapshot();
    }
    return this.tokens;
}
开发者ID:apache，项目名称:incubator-netbeans，代码行数:11，代码来源:AbstractSourceFileObject.java

示例11: testNextToken2

import org.netbeans.api.lexer.TokenHierarchy; //导入方法依赖的package包/类
public void testNextToken2() {
    String text = "abc @foo xyz\[email protected]";
    
    TokenHierarchy<?> hi = TokenHierarchy.create(text, JavadocTokenId.language());
    TokenSequence<?> ts = hi.tokenSequence();
    LexerTestUtilities.assertNextTokenEquals(ts, JavadocTokenId.IDENT, "abc");
    LexerTestUtilities.assertNextTokenEquals(ts, JavadocTokenId.OTHER_TEXT, " @");
    LexerTestUtilities.assertNextTokenEquals(ts, JavadocTokenId.IDENT, "foo");
    LexerTestUtilities.assertNextTokenEquals(ts, JavadocTokenId.OTHER_TEXT, " ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavadocTokenId.IDENT, "xyz");
    LexerTestUtilities.assertNextTokenEquals(ts, JavadocTokenId.OTHER_TEXT, "\n");
    LexerTestUtilities.assertNextTokenEquals(ts, JavadocTokenId.TAG, "@deprecated");
}
开发者ID:apache，项目名称:incubator-netbeans，代码行数:14，代码来源:JavadocLexerTest.java

示例12: testNoExoticIdentifiers

import org.netbeans.api.lexer.TokenHierarchy; //导入方法依赖的package包/类
public void testNoExoticIdentifiers() {
    String text = "a #\" \"";
    InputAttributes attr = new InputAttributes();
    attr.setValue(JavaTokenId.language(), "version", Integer.valueOf(5), true);
    TokenHierarchy<?> hi = TokenHierarchy.create(text, false, JavaTokenId.language(), EnumSet.of(JavaTokenId.WHITESPACE), attr);
    TokenSequence<?> ts = hi.tokenSequence();

    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.IDENTIFIER, "a");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.ERROR, "#");
    LexerTestUtilities.assertNextTokenEquals(ts, JavaTokenId.STRING_LITERAL, "\" \"");
}
开发者ID:apache，项目名称:incubator-netbeans，代码行数:12，代码来源:JavaLexerBatchTest.java

示例13: parse

import org.netbeans.api.lexer.TokenHierarchy; //导入方法依赖的package包/类
public static Collection<? extends HintWrapper> parse(@NonNull FileObject file, String spec) {
    TokenHierarchy<?> h = TokenHierarchy.create(spec, DeclarativeHintTokenId.language());
    TokenSequence<DeclarativeHintTokenId> ts = h.tokenSequence(DeclarativeHintTokenId.language());
    List<HintWrapper> result = new LinkedList<HintWrapper>();
    Result parsed = new DeclarativeHintsParser().parse(file, spec, ts);

    for (HintTextDescription d : parsed.hints) {
        result.add(new HintWrapper(spec, parsed, d));
    }

    return result;
}
开发者ID:apache，项目名称:incubator-netbeans，代码行数:13，代码来源:HintWrapper.java

示例14: testNextBrokenHTML3

import org.netbeans.api.lexer.TokenHierarchy; //导入方法依赖的package包/类
public void testNextBrokenHTML3() {
    String text = "<code <code\n @param";
    
    TokenHierarchy<?> hi = TokenHierarchy.create(text, JavadocTokenId.language());
    TokenSequence<?> ts = hi.tokenSequence();
    LexerTestUtilities.assertNextTokenEquals(ts, JavadocTokenId.HTML_TAG, "<code ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavadocTokenId.HTML_TAG, "<code");
    LexerTestUtilities.assertNextTokenEquals(ts, JavadocTokenId.OTHER_TEXT, "\n ");
    LexerTestUtilities.assertNextTokenEquals(ts, JavadocTokenId.TAG, "@param");
}
开发者ID:apache，项目名称:incubator-netbeans，代码行数:11，代码来源:JavadocLexerTest.java

示例15: performTest

import org.netbeans.api.lexer.TokenHierarchy; //导入方法依赖的package包/类
private void performTest(String code, Map<String, String> goldenGlobalOptions, String goldenImportsBlock, Collection<StringHintDescription> goldenHints, Collection<String> goldenBlocks) throws Exception {
    TokenHierarchy<?> h = TokenHierarchy.create(code, DeclarativeHintTokenId.language());
    FileObject file = FileUtil.createData(new File(getWorkDir(), "Test.java"));
    TestUtilities.copyStringToFile(file, code);
    Result parsed = new DeclarativeHintsParser().parse(file, code, h.tokenSequence(DeclarativeHintTokenId.language()));
    List<StringHintDescription> real = new LinkedList<StringHintDescription>();

    for (HintTextDescription hint : parsed.hints) {
        real.add(StringHintDescription.create(code, hint));
    }

    assertEquals(goldenGlobalOptions, parsed.options);
    
    if (goldenImportsBlock != null) {
        assertNotNull(parsed.importsBlock);
        assertEquals(goldenImportsBlock, code.substring(parsed.importsBlock[0], parsed.importsBlock[1]));
    } else {
        assertNull(parsed.importsBlock);
    }

    assertEquals(goldenHints, real);
    if (goldenBlocks != null) {
        assertNotNull(parsed.blocks);

        List<String> realBlocks = new LinkedList<String>();
        
        for (int[] span : parsed.blocks) {
            realBlocks.add(code.substring(span[0], span[1]));
        }
    } else {
        assertNull(parsed.blocks);
    }
}
开发者ID:apache，项目名称:incubator-netbeans，代码行数:34，代码来源:DeclarativeHintsParserTest.java


注：本文中的org.netbeans.api.lexer.TokenHierarchy.create方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。