Java TokenStream类代码示例

import org.antlr.v4.runtime.TokenStream; //导入依赖的package包/类
/**
 * @throws InvalidQueryException if the given query contains invalid characters or is incomplete
 */
public SqlGrammarParser.SelectStatementContext parse(String query) {
    log.trace("About to parse [{}]", query);
    long now = currentTimeMillis();

    CharStream input = CharStreams.fromString(query);
    SqlGrammarLexer lexer = new SqlGrammarLexer(input);
    TokenStream tokenStream = new CommonTokenStream(lexer);
    SqlGrammarParser parser = new SqlGrammarParser(tokenStream);

    parser.removeErrorListeners();
    parser.addErrorListener(new RaiseExceptionErrorListener());

    SqlGrammarParser.SelectStatementContext result = parser.selectStatement();

    log.trace("Successfully parsed [{}] into [{}] in [{}ms]",
            query, result.toStringTree(parser), currentTimeMillis() - now);

    return result;
}
 

import org.antlr.v4.runtime.TokenStream; //导入依赖的package包/类
/**
 * @param args the command line arguments
 * @throws java.io.IOException
 * @throws java.net.URISyntaxException
 */
public static void main(String[] args) throws IOException, URISyntaxException {
    final String entryPoint;
    final URL res;
    switch (args.length) {
    case 2:
        res = Paths.get(args[0]).toUri().toURL();
        entryPoint = args[1];
        break;
    default:
        System.err.println("Supply two parameters in the following order:\n- file name of the main function\n- name of the main function\n\nFor example: hello.ct main:argc:argv");
        return;
    }
    final CharStream inp = CharStreams.fromStream(res.openStream());
    final GrammarLexer lex = new GrammarLexer(inp);
    final TokenStream toks = new CommonTokenStream(lex);
    final GrammarParser parser = new GrammarParser(toks);
    System.out.println(new Translator(res.toURI()).generate(parser.program(), entryPoint));
}
 

import org.antlr.v4.runtime.TokenStream; //导入依赖的package包/类
@Override
protected void reportNoViableAlternative(Parser recognizer, NoViableAltException e) {
	// change error message from default implementation
	TokenStream tokens = recognizer.getInputStream();
	String input;
	if (tokens != null) {
		if (e.getStartToken().getType() == Token.EOF) {
			input = "the end";
		} else {
			input = escapeWSAndQuote(tokens.getText(e.getStartToken(), e.getOffendingToken()));
		}
	} else {
		input = escapeWSAndQuote("<unknown input>");
	}
	String msg = "inadmissible input at " + input;
	recognizer.notifyErrorListeners(e.getOffendingToken(), msg, e);
}
 

import org.antlr.v4.runtime.TokenStream; //导入依赖的package包/类
/**
 "If an operator has whitespace on the right side only, it is treated as a
 postfix unary operator. As an example, the ++ operator in a++ b is treated
 as a postfix unary operator."
 "If an operator has no whitespace on the left but is followed immediately
 by a dot (.), it is treated as a postfix unary operator. As an example,
 the ++ operator in a++.b is treated as a postfix unary operator (a++ .b
 rather than a ++ .b)."
 */
public static boolean isPostfixOp(TokenStream tokens) {
	int stop = getLastOpTokenIndex(tokens);
	if ( stop==-1 ) return false;

	int start = tokens.index();
	Token prevToken = tokens.get(start-1); // includes hidden-channel tokens
	Token nextToken = tokens.get(stop+1);
	boolean prevIsWS = isLeftOperatorWS(prevToken);
	boolean nextIsWS = isRightOperatorWS(nextToken);
	boolean result =
		!prevIsWS && nextIsWS ||
		!prevIsWS && nextToken.getType()==SwiftParser.DOT;
	String text = tokens.getText(Interval.of(start, stop));
	//System.out.println("isPostfixOp: '"+prevToken+"','"+text+"','"+nextToken+"' is "+result);
	return result;
}
 

import org.antlr.v4.runtime.TokenStream; //导入依赖的package包/类
public EObject parse(final String typeExpression, final Scope scope) {
    final CharStream charStream = CharStreams.fromString(typeExpression);
    final TypeExpressionLexer lexer = new TypeExpressionLexer(charStream);
    final TokenStream tokenStream = new CommonTokenStream(lexer);
    final TypeExpressionParser typeExpressionParser = new TypeExpressionParser(tokenStream);

    lexer.removeErrorListeners();
    typeExpressionParser.removeErrorListeners();

    final ParserErrorCollector errorCollector = new ParserErrorCollector();
    lexer.addErrorListener(errorCollector);
    typeExpressionParser.addErrorListener(errorCollector);

    final TypeExpressionParser.Type_exprContext typeExpr = typeExpressionParser.type_expr();

    final EObject anyType = new TypeExpressionBuilder(scope, ARRAY_TYPE).visit(typeExpr);
    scope.getResource().getErrors().addAll(errorCollector.getErrors());

    return anyType;
}
 

import org.antlr.v4.runtime.TokenStream; //导入依赖的package包/类
@Override
protected void doLoad(final InputStream inputStream, final Map<?, ?> options) throws IOException {
    final BufferedInputStream bufferedInputStream = new BufferedInputStream(inputStream);
    final Optional<AbstractConstructor> optionalRootConstructor = getRootConstructor(bufferedInputStream);

    if (optionalRootConstructor.isPresent()) {
        final AbstractConstructor rootConstructor = optionalRootConstructor.get();
        final RAMLCustomLexer lexer = new RAMLCustomLexer(uri, getURIConverter());
        final TokenStream tokenStream = new CommonTokenStream(lexer);
        final RAMLParser parser = new RAMLParser(tokenStream);

        parser.removeErrorListeners();
        final ParserErrorCollector errorCollector = new ParserErrorCollector();
        parser.addErrorListener(errorCollector);

        try {
            rootConstructor.construct(parser, resourceScope);
            validate();
        } catch (final Exception e) {
            getErrors().addAll(errorCollector.getErrors());
            throw e;
        }
    }
}
 

import org.antlr.v4.runtime.TokenStream; //导入依赖的package包/类
@Ignore
@Test
public void api() {
    final File apiFile = new File("/Users/mkoester/Development/commercetools-api-reference/api.raml");
    assumeTrue(apiFile.exists());

    final URI apiUri = URI.createURI(apiFile.toURI().toString());
    final URIConverter uriConverter = new RamlResourceSet().getURIConverter();
    final RAMLCustomLexer lexer = new RAMLCustomLexer(apiUri, uriConverter);
    final TokenStream tokenStream = new CommonTokenStream(lexer);

    final RAMLParser parser = new RAMLParser(tokenStream);
    final Resource resource = new RamlResourceSet().createResource(apiUri);

    final Scope scope = Scope.of(resource);
    final TypeDeclarationResolver resolver = new TypeDeclarationResolver();
    resolver.resolve(parser.api(), scope);

    assertThat(resource.getErrors()).isEmpty();
}
 

import org.antlr.v4.runtime.TokenStream; //导入依赖的package包/类
public static boolean isClassName(TokenStream _input) {
  try {
    int i=1;
    Token token = _input.LT(i);
    while (token!=null && i < _input.size() && _input.LT(i+1).getType() == GroovyParser.DOT) {
      i = i + 2;
      token = _input.LT(i);
    }
    if(token==null)
      return false;
    // TODO here
    return Character.isUpperCase(Character.codePointAt(token.getText(), 0));
  } catch(Exception e) {
    e.printStackTrace();
  }
  
  return false;
}
 

import org.antlr.v4.runtime.TokenStream; //导入依赖的package包/类
public static <L extends Lexer, P extends Parser> P newParser(
    Function<CharStream, L> lexerFactory,
    Function<TokenStream, P> parserFactory,
    String input,
    boolean useBailErrorStrategy,
    boolean removeErrorListeners) {
  CharStream charStream = new ANTLRInputStream(input);
  L lexer = lexerFactory.apply(charStream);
  if (removeErrorListeners) {
    lexer.removeErrorListeners();
  }

  TokenStream tokenStream = new CommonTokenStream(lexer);
  P parser = parserFactory.apply(tokenStream);
  if (useBailErrorStrategy) {
    parser.setErrorHandler(new BailErrorStrategy());
  }
  if (removeErrorListeners) {
    parser.removeErrorListeners();
  }

  return parser;
}
 

import org.antlr.v4.runtime.TokenStream; //导入依赖的package包/类
/**
 * Parse an interval, for example <tt>[1,-]</tt> or <tt>-</tt> (a wildcard) or <tt>[1,4]</tt>.
 * Only fixed values are allowed, no variables.
 *
 * @param intervalAsString the string to be parsed.
 * @return a LowerBoundedInterval as the runtime representation of interval strings.
 * @throws ParseException in case the string doesn't fit the given fixed-interval grammar.
 */
public static LowerBoundedInterval parse(String intervalAsString) throws ParseException {
  CharStream charStream = new ANTLRInputStream(intervalAsString);
  CellExpressionLexer lexer = new CellExpressionLexer(charStream);
  TokenStream tokens = new CommonTokenStream(lexer);
  CellExpressionParser parser = new CellExpressionParser(tokens);
  parser.removeErrorListeners();
  parser.addErrorListener(new ThrowingErrorListener());
  try {
    CellExpressionParser.Fixed_intervalContext ctx = parser.fixed_interval();
    if (ctx == null) {
      throw new ParseException(0, 0, "Expected fixed interval");
    }
    return INSTANCE.visit(ctx);
  } catch (ParseRuntimeException runtimeException) {
    throw runtimeException.getParseException();
  }
}
 

import org.antlr.v4.runtime.TokenStream; //导入依赖的package包/类
public static AstBuilder createAstBuilder(CompilationUnit source,TokenStream tokens){
    KalangParser p = new KalangParser(tokens);
    AstBuilder sp = new AstBuilder(source, p);
    p.setErrorHandler(new DefaultErrorStrategy() {

        @Override
        public void reportError(Parser recognizer, RecognitionException e) {
            String msg = AntlrErrorString.exceptionString(recognizer, e);
            Token end = e.getOffendingToken();
            Token start;
            RuleContext ctx = e.getCtx();
            if(ctx instanceof ParserRuleContext){
                start = ((ParserRuleContext) ctx).getStart();
            }else{
                start = end;
            }
            sp.getDiagnosisReporter().report(Diagnosis.Kind.ERROR, msg,start,end);
        }
    });
    return sp;
}
 

import org.antlr.v4.runtime.TokenStream; //导入依赖的package包/类
private void compileFiles(List<RawFile> files, OOPSourceCodeModel srcModel, List<String> projectFileTypes) {
    for (RawFile file : files) {
        try {
            CharStream charStream = new ANTLRInputStream(file.content());
            GolangLexer lexer = new GolangLexer(charStream);
            TokenStream tokens = new CommonTokenStream(lexer);
            GolangParser parser = new GolangParser(tokens);
            SourceFileContext sourceFileContext = parser.sourceFile();
            parser.setErrorHandler(new BailErrorStrategy());
            parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
            ParseTreeWalker walker = new ParseTreeWalker();
            GolangBaseListener listener = new GoLangTreeListener(srcModel, projectFileTypes, file);
            walker.walk(listener, sourceFileContext);
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}
 

import org.antlr.v4.runtime.TokenStream; //导入依赖的package包/类
private SyntaxErrorListener getSyntaxTreeErrors(String iFlowResource) {
    InputStream inputStream = null;

    try {
        File file = new File(getClass().getResource(iFlowResource).getFile());
        inputStream = new FileInputStream(file);

        CharStream cs = new ANTLRInputStream(inputStream);
        TokenStream tokenStream = new CommonTokenStream(new WUMLLexer(cs));
        WUMLParser parser = new WUMLParser(tokenStream);
        SyntaxErrorListener errorListener = new SyntaxErrorListener();
        parser.addErrorListener(errorListener);
        ParseTree tree = parser.sourceFile();
        return errorListener;

    } catch (Exception e) {
        return null;
    }
}
 

import org.antlr.v4.runtime.TokenStream; //导入依赖的package包/类
public static Program parse(String source) {
	RankPLLexer lexer = new RankPLLexer(new ANTLRInputStream(source));
	TokenStream tokens = new CommonTokenStream(lexer);
	RankPLParser parser = new RankPLParser(tokens);
	parser.setErrorHandler(new BailErrorStrategy());
	ConcreteParser classVisitor = new ConcreteParser();

	// Parse
	Program program = null;
	try {
		program = (Program) classVisitor.visit(parser.program());
	} catch (ParseCancellationException e) {
		System.out.println("Syntax error");
		lexer = new RankPLLexer(new ANTLRInputStream(source));
		tokens = new CommonTokenStream(lexer);
		parser = new RankPLParser(tokens);
		classVisitor = new ConcreteParser();
		try {
			program = (Program) classVisitor.visit(parser.program());
		} catch (Exception ex) {
			// Ignore
		}
		return null;
	}
	return program;
}
 

import org.antlr.v4.runtime.TokenStream; //导入依赖的package包/类
public ParseTreeDOMSerializer(List<String> ruleNames, Map<Integer, String> invTokenMap, TokenStream tokenStream) {
  	this.tokenStream = tokenStream;
      this.ruleNames = ruleNames;
      this.invTokenMap = invTokenMap;
      nodeStack = new Stack<Element>();
      
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
Document document = null;
try {
	DocumentBuilder builder = factory.newDocumentBuilder();
	document = builder.newDocument();
	Element root = (Element) document.createElement("ast");
	document.appendChild(root);
	nodeStack.push(root);
	domDoc = document;
} catch (Exception pce) {
	pce.printStackTrace();
}
  }