本文整理汇总了Java中org.antlr.runtime.CommonTokenStream.getTokens方法的典型用法代码示例。如果您正苦于以下问题：Java CommonTokenStream.getTokens方法的具体用法？Java CommonTokenStream.getTokens怎么用？Java CommonTokenStream.getTokens使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.antlr.runtime.CommonTokenStream的用法示例。
在下文中一共展示了CommonTokenStream.getTokens方法的8个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: getTokens
import org.antlr.runtime.CommonTokenStream; //导入方法依赖的package包/类
public static List<Token> getTokens(String sourceCode, String language)
throws Exception {
Lexer lexer = null;
if (language.compareTo("C++") == 0) {
lexer = new PlagiCOJCppLexer(new ANTLRStringStream(sourceCode));
} else if (language.compareTo("Java") == 0) {
lexer = new PlagiCOJJavaLexer(new ANTLRStringStream(sourceCode));
} else if (language.compareTo("C#") == 0) {
lexer = new PlagiCOJCSharpLexer(new ANTLRStringStream(sourceCode));
} else if (language.compareTo("C") == 0) {
lexer = new PlagiCOJCLexer(new ANTLRStringStream(sourceCode));
} else if (language.compareTo("Python") == 0) {
lexer = new PlagiCOJPythonLexer(new ANTLRStringStream(sourceCode));
} else if (language.compareTo("Pascal") == 0) {
lexer = new PlagiCOJPascalLexer(new ANTLRStringStream(sourceCode));
} else {
throw new PlagiCOJUnsupportedLanguageException();
}
CommonTokenStream tokens = new CommonTokenStream(lexer);
return tokens.getTokens();
}
示例2: testLexerOutput
import org.antlr.runtime.CommonTokenStream; //导入方法依赖的package包/类
public void testLexerOutput() throws IOException
{
String str = "~woof^2";
CharStream input = new ANTLRInputStream(new ByteArrayInputStream(str.getBytes("UTF-8")));
FTSLexer lexer = new FTSLexer(input);
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
for(CommonToken token : (List<CommonToken>)tokenStream.getTokens())
{
System.out.println(token.toString());
}
}
示例3: testLexerError_01
import org.antlr.runtime.CommonTokenStream; //导入方法依赖的package包/类
@Test public void testLexerError_01() throws Exception {
String model = "a /* incomplete comment *";
InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
lexer.setCharStream(new ANTLRStringStream(model));
CommonTokenStream stream = new CommonTokenStream(lexer);
@SuppressWarnings("unchecked")
List<CommonToken> tokens = stream.getTokens();
assertEquals(tokens.toString(), 3, tokens.size());
assertEquals("a", tokens.get(0).getText());
assertEquals(" ", tokens.get(1).getText());
assertEquals("/* incomplete comment *", tokens.get(2).getText());
assertEquals(0, tokens.get(2).getType());
}
示例4: testLexerError_02
import org.antlr.runtime.CommonTokenStream; //导入方法依赖的package包/类
@Test public void testLexerError_02() throws Exception {
String model = "a 'incomplete string";
InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
lexer.setCharStream(new ANTLRStringStream(model));
CommonTokenStream stream = new CommonTokenStream(lexer);
@SuppressWarnings("unchecked")
List<CommonToken> tokens = stream.getTokens();
assertEquals(tokens.toString(), 3, tokens.size());
assertEquals("a", tokens.get(0).getText());
assertEquals(" ", tokens.get(1).getText());
assertEquals("'incomplete string", tokens.get(2).getText());
assertEquals(0, tokens.get(2).getType());
}
示例5: testLexerError_03
import org.antlr.runtime.CommonTokenStream; //导入方法依赖的package包/类
@Test public void testLexerError_03() throws Exception {
String model = "a '\\ incomplete string with bad escape sequence";
InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
lexer.setCharStream(new ANTLRStringStream(model));
CommonTokenStream stream = new CommonTokenStream(lexer);
@SuppressWarnings("unchecked")
List<CommonToken> tokens = stream.getTokens();
assertEquals(tokens.toString(), 3, tokens.size());
assertEquals("a", tokens.get(0).getText());
assertEquals(" ", tokens.get(1).getText());
assertEquals("'\\ incomplete string with bad escape sequence", tokens.get(2).getText());
assertEquals(0, tokens.get(2).getType());
}
示例6: testLexerError_04
import org.antlr.runtime.CommonTokenStream; //导入方法依赖的package包/类
@Test public void testLexerError_04() throws Exception {
String model = "a 'incomplete string with bad escape sequence \\";
InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
lexer.setCharStream(new ANTLRStringStream(model));
CommonTokenStream stream = new CommonTokenStream(lexer);
@SuppressWarnings("unchecked")
List<CommonToken> tokens = stream.getTokens();
assertEquals(tokens.toString(), 3, tokens.size());
assertEquals("a", tokens.get(0).getText());
assertEquals(" ", tokens.get(1).getText());
assertEquals("'incomplete string with bad escape sequence \\", tokens.get(2).getText());
assertEquals(0, tokens.get(2).getType());
}
示例7: testLexerError_05
import org.antlr.runtime.CommonTokenStream; //导入方法依赖的package包/类
@Test public void testLexerError_05() throws Exception {
String model = "a 'incomplete string \\'";
InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
lexer.setCharStream(new ANTLRStringStream(model));
CommonTokenStream stream = new CommonTokenStream(lexer);
@SuppressWarnings("unchecked")
List<CommonToken> tokens = stream.getTokens();
assertEquals(tokens.toString(), 3, tokens.size());
assertEquals("a", tokens.get(0).getText());
assertEquals(" ", tokens.get(1).getText());
assertEquals("'incomplete string \\'", tokens.get(2).getText());
assertEquals(0, tokens.get(2).getType());
}
示例8: testLexerError_06
import org.antlr.runtime.CommonTokenStream; //导入方法依赖的package包/类
@Test public void testLexerError_06() throws Exception {
String model = "a '";
InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
lexer.setCharStream(new ANTLRStringStream(model));
CommonTokenStream stream = new CommonTokenStream(lexer);
@SuppressWarnings("unchecked")
List<CommonToken> tokens = stream.getTokens();
assertEquals(tokens.toString(), 3, tokens.size());
assertEquals("a", tokens.get(0).getText());
assertEquals(" ", tokens.get(1).getText());
assertEquals("'", tokens.get(2).getText());
assertEquals(InternalSimpleExpressionsTestLanguageParser.RULE_ANY_OTHER, tokens.get(2).getType());
}