Java TokenStream類代碼示例

import org.antlr.runtime.TokenStream; //導入依賴的package包/類
/**
 * Just call a parser method in {@link CqlParser} - does not do any error handling.
 */
public static <R> R parseAnyUnhandled(CQLParserFunction<R> parserFunction, String input) throws RecognitionException
{
    // Lexer and parser
    ErrorCollector errorCollector = new ErrorCollector(input);
    CharStream stream = new ANTLRStringStream(input);
    CqlLexer lexer = new CqlLexer(stream);
    lexer.addErrorListener(errorCollector);

    TokenStream tokenStream = new CommonTokenStream(lexer);
    CqlParser parser = new CqlParser(tokenStream);
    parser.addErrorListener(errorCollector);

    // Parse the query string to a statement instance
    R r = parserFunction.parse(parser);

    // The errorCollector has queue up any errors that the lexer and parser may have encountered
    // along the way, if necessary, we turn the last error into exceptions here.
    errorCollector.throwFirstSyntaxError();

    return r;
}
 

import org.antlr.runtime.TokenStream; //導入依賴的package包/類
/**
 * Returns true if there was an unexpected EOL.
 */
public static boolean hasDisallowedEOL(Callback callback) {
	TokenStream input = callback.getInput();
	Token lt = input.LT(1);

	// Start on the position before the current token and scan backwards off channel tokens until the previous on
	// channel token.
	for (int ix = lt.getTokenIndex() - 1; ix > 0; ix--) {
		lt = input.get(ix);
		if (lt.getChannel() == Token.DEFAULT_CHANNEL) {
			// On channel token found: stop scanning.
			break;
		} else if (isSemicolonEquivalent(lt)) {
			return true;
		}
	}
	return false;
}
 

import org.antlr.runtime.TokenStream; //導入依賴的package包/類
/** Creates a parser for a given term, generating trees of this kind. */
public <P extends Parser> P createParser(Class<P> parserType, I info, String term) {
    try {
        // find the lexer type
        String parserName = parserType.getName();
        String lexerName = parserName.substring(0, parserName.indexOf("Parser"))
            .concat("Lexer");
        @SuppressWarnings("unchecked") Class<? extends Lexer> lexerType =
            (Class<? extends Lexer>) Class.forName(lexerName);
        Lexer lexer = createLexer(lexerType, info, term);
        // instantiate the parser
        CommonTokenStream tokenStream = new CommonTokenStream(lexer);
        Constructor<P> parserConstructor = parserType.getConstructor(TokenStream.class);
        P result = parserConstructor.newInstance(tokenStream);
        Method adaptorSetter = parserType.getMethod("setTreeAdaptor", TreeAdaptor.class);
        adaptorSetter.invoke(result, new ParseTreeAdaptor<>(this, info, tokenStream));
        callInitialise(result, info);
        return result;
    } catch (Exception e) {
        throw toRuntime(e);
    }
}
 

import org.antlr.runtime.TokenStream; //導入依賴的package包/類
@Test
public void test1() throws RecognitionException {
    CharStream input = new ANTLRStringStream("version(1.1.0)\n" + "minVer(1.1.0)\n"
            + "[ The Admin API is used to manipulate and access the low level entities in Rapture. Typically the methods in this API\n"
            + "are only used during significant setup events in a Rapture environment.]\n" + "api(Admin) {\n"
            + "    [This method restores a user that has been deleted]\n" + "    @entitle=/admin/main\n"
            + "    @public Boolean restoreUser(String userName);\n" + "}\n" + "[A return value from a native query]\n"
            + "type RaptureQueryResult(@package=rapture.common) {\n" + "    List(JsonContent) rows;\n" + "}\n");
    TLexer lexer = new TLexer(input);
    TokenStream tokenInputStream = new CommonTokenStream(lexer);
    TParser parser = new TParser(tokenInputStream);
    hmxdef_return returnVal = parser.hmxdef();
    System.out.println("Done " + returnVal.getTree().toStringTree());

    TreeNodeStream treeInput = new CommonTreeNodeStream(returnVal.getTree());
    TTree walker = new TTree(treeInput);
    walker.setTemplateLib(TemplateRepo.getApiTemplates("Java"));
    com.incapture.rapgen.TTree.hmxdef_return walkerResult = walker.hmxdef();
    System.out.println("Done, result=" + walkerResult.toString());
}
 

import org.antlr.runtime.TokenStream; //導入依賴的package包/類
@Test
public void test2() throws RecognitionException {
    CharStream input = new ANTLRStringStream("version(1.1.0)\n" + "minVer(1.1.0)\n"
            + "[ The Admin API is used to manipulate and access the low level entities in Rapture. Typically the methods in this API\n"
            + "are only used during significant setup events in a Rapture environment.]\n" + "api(Admin) {\n"
            + "    [This method restores a user that has been deleted]\n" + "    @entitle=/admin/main\n"
            + "    @public Boolean restoreUser(String userName);\n" + "}\n" + "[A return value from a native query]\n"
            + "type RaptureQueryResult(@package=rapture.common) {\n" + "    List(JsonContent) rows;\n" + "}\n");
    TLexer lexer = new TLexer(input);
    TokenStream tokenInputStream = new CommonTokenStream(lexer);
    TParser parser = new TParser(tokenInputStream);
    hmxdef_return returnVal = parser.hmxdef();
    System.out.println("Done " + returnVal.getTree().toStringTree());

    TreeNodeStream treeInput = new CommonTreeNodeStream(returnVal.getTree());
    TTree walker = new TTree(treeInput);
    walker.setTemplateLib(TemplateRepo.getApiTemplates("Java"));
    apiGen_return walkerResult = walker.apiGen();
    System.out.println("Done, result=" + walkerResult.toString());
}
 

import org.antlr.runtime.TokenStream; //導入依賴的package包/類
@Test
public void test3() throws RecognitionException {
    CharStream input = new ANTLRStringStream("version(1.1.0)\n" + "minVer(1.1.0)\n"
            + "[ The Admin API is used to manipulate and access the low level entities in Rapture. Typically the methods in this API\n"
            + "are only used during significant setup events in a Rapture environment.]\n" + "api(Admin) {\n"
            + "    [This method restores a user that has been deleted]\n" + "    @entitle=/admin/main\n"
            + "    @public Boolean restoreUser(String userName);\n" + "}\n" + "[A return value from a native query]\n"
            + "type RaptureQueryResult(@package=rapture.common) {\n" + "    List(JsonContent) rows;\n" + "}\n");
    TLexer lexer = new TLexer(input);
    TokenStream tokenInputStream = new CommonTokenStream(lexer);
    TParser parser = new TParser(tokenInputStream);
    hmxdef_return returnVal = parser.hmxdef();
    System.out.println("Done " + returnVal.getTree().toStringTree());

    TreeNodeStream treeInput = new CommonTreeNodeStream(returnVal.getTree());
    TTree walker = new TTree(treeInput);
    walker.setTemplateLib(TemplateRepo.getSdkTemplates("Java"));
    try {
        walker.sdkGen();
        fail("This should have failed, sdk name is required but not present in code");
    } catch (IllegalArgumentException e) {
        assertTrue(e.getMessage().contains("SDK Name"));
    }
}
 

import org.antlr.runtime.TokenStream; //導入依賴的package包/類
@Test
public void test4() throws RecognitionException {
    CharStream input = new ANTLRStringStream("sdk(alan)\n" + "version(0.0.1)\n" + "minVer(0.0.1)\n"
            + "[ The Admin API is used to manipulate and access the low level entities in Rapture. Typically the methods in this API\n"
            + "are only used during significant setup events in a Rapture environment.]\n" + "api(Admin) {\n"
            + "    [This method restores a user that has been deleted]\n" + "    @entitle=/admin/main\n"
            + "    @public Boolean restoreUser(String userName);\n" + "}\n" + "[A return value from a native query]\n"
            + "type RaptureQueryResult(@package=rapture.common) {\n" + "    List(JsonContent) rows;\n" + "}\n");
    TLexer lexer = new TLexer(input);
    TokenStream tokenInputStream = new CommonTokenStream(lexer);
    TParser parser = new TParser(tokenInputStream);
    hmxdef_return returnVal = parser.hmxdef();
    System.out.println("Done " + returnVal.getTree().toStringTree());

    TreeNodeStream treeInput = new CommonTreeNodeStream(returnVal.getTree());
    TTree walker = new TTree(treeInput);
    walker.setTemplateLib(TemplateRepo.getSdkTemplates("Java"));
    walker.sdkGen();
}
 

import org.antlr.runtime.TokenStream; //導入依賴的package包/類
@Test
public void test5() throws RecognitionException {
    CharStream input = new ANTLRStringStream("sdk(alan)\n" + "version(0.0.1)\n" + "minVer(0.0.1)\n"
            + "[ The Admin API is used to manipulate and access the low level entities in Rapture. Typically the methods in this API\n"
            + "are only used during significant setup events in a Rapture environment.]\n" + "api(Admin) {\n"
            + "    [This method restores a user that has been deleted]\n" + "    @entitle=/admin/main\n"
            + "    @public Boolean restoreUser(String userName);\n" + "}\n" + "[A return value from a native query]\n"
            + "type RaptureQueryResult(@package=rapture.common) {\n" + "    List(JsonContent) rows;\n" + "}\n");
    TLexer lexer = new TLexer(input);
    TokenStream tokenInputStream = new CommonTokenStream(lexer);
    TParser parser = new TParser(tokenInputStream);
    hmxdef_return returnVal = parser.hmxdef();
    System.out.println("Done " + returnVal.getTree().toStringTree());

    TreeNodeStream treeInput = new CommonTreeNodeStream(returnVal.getTree());
    TTree walker = new TTree(treeInput);
    walker.setTemplateLib(TemplateRepo.getApiTemplates("Java"));
    try {
        walker.apiGen();
        fail("This should have failed, sdk name is not expected but found");
    } catch (IllegalArgumentException e) {
        assertTrue(e.getMessage().contains("SDK Name"));
    }
}
 

import org.antlr.runtime.TokenStream; //導入依賴的package包/類
@Test
public void test7() throws RecognitionException {
    CharStream input = new ANTLRStringStream("version(1.1.0)\n" + "minVer(1.1.0)\n"
            + "[ The Admin API is used to manipulate and access the low level entities in Rapture. Typically the methods in this API\n"
            + "are only used during significant setup events in a Rapture environment.]\n" + "api(Admin) {\n"
            + "    [This method restores a user that has been deleted]\n" + "    @entitle=/admin/main\n"
            + "    @public Boolean restoreUser(String userName);\n" + "}\n" + "[A return value from a native query]\n"
            + "@Storable(storagePath : {\"sys\", name}, separator=\".\")\n" +
            "type RepoConfig (@package=rapture.common.model) {\n" +
            "   String name = \"test\";\n" +
            "   String config;\n" +
            "}\n" +
            "");
    TLexer lexer = new TLexer(input);
    TokenStream tokenInputStream = new CommonTokenStream(lexer);
    TParser parser = new TParser(tokenInputStream);
    hmxdef_return returnVal = parser.hmxdef();
    System.out.println("Done " + returnVal.getTree().toStringTree());

    TreeNodeStream treeInput = new CommonTreeNodeStream(returnVal.getTree());
    TTree walker = new TTree(treeInput);
    walker.setTemplateLib(TemplateRepo.getApiTemplates("Java"));
    apiGen_return walkerResult = walker.apiGen();
    System.out.println("Done, result=" + walkerResult.toString());
}
 

import org.antlr.runtime.TokenStream; //導入依賴的package包/類
@Test
public void testBean() throws RecognitionException {
    CharStream input = new ANTLRStringStream("version(1.1.0)\n" + "minVer(1.1.0)\n"
            + "[ The Admin API is used to manipulate and access the low level entities in Rapture. Typically the methods in this API\n"
            + "are only used during significant setup events in a Rapture environment.]\n" + "api(Admin) {\n"
            + "    [This method restores a user that has been deleted]\n" + "    @entitle=/admin/main\n"
            + "    @public Boolean restoreUser(String userName);\n" + "}\n" +
            "[A Graph node]\n" +
            "@Bean\n" +
            "type Node(@package=rapture.common.dp) {\n" +
            "    String nodeId; //this is not a URI, just a String id\n" +
            "    List<XFer> xferValues;\n" +
            "}\n" +
            "");
    TLexer lexer = new TLexer(input);
    TokenStream tokenInputStream = new CommonTokenStream(lexer);
    TParser parser = new TParser(tokenInputStream);
    hmxdef_return returnVal = parser.hmxdef();
    System.out.println("Done " + returnVal.getTree().toStringTree());

    TreeNodeStream treeInput = new CommonTreeNodeStream(returnVal.getTree());
    TTree walker = new TTree(treeInput);
    walker.setTemplateLib(TemplateRepo.getApiTemplates("Java"));
    apiGen_return walkerResult = walker.apiGen();
    System.out.println("Done, result=" + walkerResult.toString());
}
 

import org.antlr.runtime.TokenStream; //導入依賴的package包/類
@Test
public void test6() throws RecognitionException {
    System.out.println("6");
    CharStream input = new ANTLRStringStream("[ This is a mailbox message, usually posted by an external user]\n" + "@Addressable(scheme = MAILBOX)\n"
            + "@Storable(storagePath : {authority , documentPath , id} )\n" + "type RaptureMailMessage(@package=rapture.common.model) {\n"
            + "   String id;\n" + "   String authority;\n" + "   String category;\n" + "   String content;\n" + "   Date when;\n" + "   String who;\n"
            + "}\n" + "");
    TLexer lexer = new TLexer(input);
    TokenStream tokenInputStream = new CommonTokenStream(lexer);
    TParser parser = new TParser(tokenInputStream);
    typeExpr_return returnVal = parser.typeExpr();
    System.out.println("Done " + returnVal.getTree().toStringTree());

    TreeNodeStream treeInput = new CommonTreeNodeStream(returnVal.getTree());
    TTree walker = new TTree(treeInput);
    walker.setTemplateLib(TemplateRepo.getApiTemplates("Java"));
    com.incapture.rapgen.TTree.typeExpr_return walkerResult = walker.typeExpr();
    System.out.println("Done, result=" + walkerResult.toString());
}
 

import org.antlr.runtime.TokenStream; //導入依賴的package包/類
@Test
public void test1() throws RecognitionException, IOException {
    CharStream input = new ANTLRStringStream("version(1.1.0)\n" + "minVer(1.1.0)\n"
            + "[ The Admin API is used to manipulate and access the low level entities in Rapture. Typically the methods in this API\n"
            + "are only used during significant setup events in a Rapture environment.]\n" + "api(Admin) {\n"
            + "    [This method restores a user that has been deleted]\n" + "    @entitle=/admin/main\n"
            + "    @public Boolean restoreUser(String userName);\n" + "}\n" + "[A return value from a native query]\n"
            + "type RaptureQueryResult(@package=rapture.common) {\n" + "    List(JsonContent) rows;\n" + "}\n");
    TLexer lexer = new TLexer(input);
    TokenStream tokenInputStream = new CommonTokenStream(lexer);
    TParser parser = new TParser(tokenInputStream);
    hmxdef_return returnVal = parser.hmxdef();
    System.out.println("Done " + returnVal.getTree().toStringTree());

    TreeNodeStream treeInput = new CommonTreeNodeStream(returnVal.getTree());
    TTreeJS walker = new TTreeJS(treeInput);
    walker.setTemplateLib(getJsTemplate());
}
 

import org.antlr.runtime.TokenStream; //導入依賴的package包/類
@Test
public void testAddErrorListener() throws Exception
{
    SyntaxErrorCounter firstCounter = new SyntaxErrorCounter();
    SyntaxErrorCounter secondCounter = new SyntaxErrorCounter();

    CharStream stream = new ANTLRStringStream("SELECT * FORM test;");
    CqlLexer lexer = new CqlLexer(stream);

    TokenStream tokenStream = new CommonTokenStream(lexer);
    CqlParser parser = new CqlParser(tokenStream);
    parser.addErrorListener(firstCounter);
    parser.addErrorListener(secondCounter);

    parser.query();

    // ANTLR 3.5 reports 2 errors in the sentence above (missing FROM and missing EOF).
    assertTrue(firstCounter.count > 0);
    assertTrue(secondCounter.count > 0);
}
 

import org.antlr.runtime.TokenStream; //導入依賴的package包/類
@Test
public void testRemoveErrorListener() throws Exception
{
    SyntaxErrorCounter firstCounter = new SyntaxErrorCounter();
    SyntaxErrorCounter secondCounter = new SyntaxErrorCounter();

    CharStream stream = new ANTLRStringStream("SELECT * FORM test;");
    CqlLexer lexer = new CqlLexer(stream);

    TokenStream tokenStream = new CommonTokenStream(lexer);
    CqlParser parser = new CqlParser(tokenStream);
    parser.addErrorListener(firstCounter);
    parser.addErrorListener(secondCounter);
    parser.removeErrorListener(secondCounter);

    parser.query();

    assertTrue(firstCounter.count > 0);
    assertEquals(0, secondCounter.count);
}
 

import org.antlr.runtime.TokenStream; //導入依賴的package包/類
@Override
public JSONObject compile(String bqlStmt) throws RecognitionException {
  // Lexer splits input into tokens
  ANTLRStringStream input = new ANTLRStringStream(bqlStmt);
  TokenStream tokens = new CommonTokenStream(new PQLLexer(input));

  // Parser generates abstract syntax tree
  PQLParser parser = new PQLParser(tokens, _facetInfoMap);
  _parser.set(parser);
  PQLParser.statement_return ret = parser.statement();

  // Acquire parse result
  CommonTree ast = (CommonTree) ret.tree;

  JSONObject json = (JSONObject) ret.json;
  // XXX To be removed
  // printTree(ast);
  // System.out.println(">>> json = " + json.toString());
  return json;
}