本文整理匯總了Java中org.antlr.runtime.CommonTokenStream.getTokens方法的典型用法代碼示例。如果您正苦於以下問題：Java CommonTokenStream.getTokens方法的具體用法？Java CommonTokenStream.getTokens怎麽用？Java CommonTokenStream.getTokens使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類org.antlr.runtime.CommonTokenStream的用法示例。
在下文中一共展示了CommonTokenStream.getTokens方法的8個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Java代碼示例。
示例1: getTokens
import org.antlr.runtime.CommonTokenStream; //導入方法依賴的package包/類
public static List<Token> getTokens(String sourceCode, String language)
throws Exception {
Lexer lexer = null;
if (language.compareTo("C++") == 0) {
lexer = new PlagiCOJCppLexer(new ANTLRStringStream(sourceCode));
} else if (language.compareTo("Java") == 0) {
lexer = new PlagiCOJJavaLexer(new ANTLRStringStream(sourceCode));
} else if (language.compareTo("C#") == 0) {
lexer = new PlagiCOJCSharpLexer(new ANTLRStringStream(sourceCode));
} else if (language.compareTo("C") == 0) {
lexer = new PlagiCOJCLexer(new ANTLRStringStream(sourceCode));
} else if (language.compareTo("Python") == 0) {
lexer = new PlagiCOJPythonLexer(new ANTLRStringStream(sourceCode));
} else if (language.compareTo("Pascal") == 0) {
lexer = new PlagiCOJPascalLexer(new ANTLRStringStream(sourceCode));
} else {
throw new PlagiCOJUnsupportedLanguageException();
}
CommonTokenStream tokens = new CommonTokenStream(lexer);
return tokens.getTokens();
}
示例2: testLexerOutput
import org.antlr.runtime.CommonTokenStream; //導入方法依賴的package包/類
public void testLexerOutput() throws IOException
{
String str = "~woof^2";
CharStream input = new ANTLRInputStream(new ByteArrayInputStream(str.getBytes("UTF-8")));
FTSLexer lexer = new FTSLexer(input);
CommonTokenStream tokenStream = new CommonTokenStream(lexer);
for(CommonToken token : (List<CommonToken>)tokenStream.getTokens())
{
System.out.println(token.toString());
}
}
示例3: testLexerError_01
import org.antlr.runtime.CommonTokenStream; //導入方法依賴的package包/類
@Test public void testLexerError_01() throws Exception {
String model = "a /* incomplete comment *";
InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
lexer.setCharStream(new ANTLRStringStream(model));
CommonTokenStream stream = new CommonTokenStream(lexer);
@SuppressWarnings("unchecked")
List<CommonToken> tokens = stream.getTokens();
assertEquals(tokens.toString(), 3, tokens.size());
assertEquals("a", tokens.get(0).getText());
assertEquals(" ", tokens.get(1).getText());
assertEquals("/* incomplete comment *", tokens.get(2).getText());
assertEquals(0, tokens.get(2).getType());
}
示例4: testLexerError_02
import org.antlr.runtime.CommonTokenStream; //導入方法依賴的package包/類
@Test public void testLexerError_02() throws Exception {
String model = "a 'incomplete string";
InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
lexer.setCharStream(new ANTLRStringStream(model));
CommonTokenStream stream = new CommonTokenStream(lexer);
@SuppressWarnings("unchecked")
List<CommonToken> tokens = stream.getTokens();
assertEquals(tokens.toString(), 3, tokens.size());
assertEquals("a", tokens.get(0).getText());
assertEquals(" ", tokens.get(1).getText());
assertEquals("'incomplete string", tokens.get(2).getText());
assertEquals(0, tokens.get(2).getType());
}
示例5: testLexerError_03
import org.antlr.runtime.CommonTokenStream; //導入方法依賴的package包/類
@Test public void testLexerError_03() throws Exception {
String model = "a '\\ incomplete string with bad escape sequence";
InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
lexer.setCharStream(new ANTLRStringStream(model));
CommonTokenStream stream = new CommonTokenStream(lexer);
@SuppressWarnings("unchecked")
List<CommonToken> tokens = stream.getTokens();
assertEquals(tokens.toString(), 3, tokens.size());
assertEquals("a", tokens.get(0).getText());
assertEquals(" ", tokens.get(1).getText());
assertEquals("'\\ incomplete string with bad escape sequence", tokens.get(2).getText());
assertEquals(0, tokens.get(2).getType());
}
示例6: testLexerError_04
import org.antlr.runtime.CommonTokenStream; //導入方法依賴的package包/類
@Test public void testLexerError_04() throws Exception {
String model = "a 'incomplete string with bad escape sequence \\";
InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
lexer.setCharStream(new ANTLRStringStream(model));
CommonTokenStream stream = new CommonTokenStream(lexer);
@SuppressWarnings("unchecked")
List<CommonToken> tokens = stream.getTokens();
assertEquals(tokens.toString(), 3, tokens.size());
assertEquals("a", tokens.get(0).getText());
assertEquals(" ", tokens.get(1).getText());
assertEquals("'incomplete string with bad escape sequence \\", tokens.get(2).getText());
assertEquals(0, tokens.get(2).getType());
}
示例7: testLexerError_05
import org.antlr.runtime.CommonTokenStream; //導入方法依賴的package包/類
@Test public void testLexerError_05() throws Exception {
String model = "a 'incomplete string \\'";
InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
lexer.setCharStream(new ANTLRStringStream(model));
CommonTokenStream stream = new CommonTokenStream(lexer);
@SuppressWarnings("unchecked")
List<CommonToken> tokens = stream.getTokens();
assertEquals(tokens.toString(), 3, tokens.size());
assertEquals("a", tokens.get(0).getText());
assertEquals(" ", tokens.get(1).getText());
assertEquals("'incomplete string \\'", tokens.get(2).getText());
assertEquals(0, tokens.get(2).getType());
}
示例8: testLexerError_06
import org.antlr.runtime.CommonTokenStream; //導入方法依賴的package包/類
@Test public void testLexerError_06() throws Exception {
String model = "a '";
InternalSimpleExpressionsTestLanguageLexer lexer = new InternalSimpleExpressionsTestLanguageLexer();
lexer.setCharStream(new ANTLRStringStream(model));
CommonTokenStream stream = new CommonTokenStream(lexer);
@SuppressWarnings("unchecked")
List<CommonToken> tokens = stream.getTokens();
assertEquals(tokens.toString(), 3, tokens.size());
assertEquals("a", tokens.get(0).getText());
assertEquals(" ", tokens.get(1).getText());
assertEquals("'", tokens.get(2).getText());
assertEquals(InternalSimpleExpressionsTestLanguageParser.RULE_ANY_OTHER, tokens.get(2).getType());
}