Java CommonToken类代码示例

import org.antlr.v4.runtime.CommonToken; //导入依赖的package包/类
@Override
public void exitNonReserved(SqlBaseParser.NonReservedContext context)
{
    // we can't modify the tree during rule enter/exit event handling unless we're dealing with a terminal.
    // Otherwise, ANTLR gets confused an fires spurious notifications.
    if (!(context.getChild(0) instanceof TerminalNode)) {
        int rule = ((ParserRuleContext) context.getChild(0)).getRuleIndex();
        throw new AssertionError("nonReserved can only contain tokens. Found nested rule: " + ruleNames.get(rule));
    }

    // replace nonReserved words with IDENT tokens
    context.getParent().removeLastChild();

    Token token = (Token) context.getChild(0).getPayload();
    context.getParent().addChild(new CommonToken(
            new Pair<>(token.getTokenSource(), token.getInputStream()),
            SqlBaseLexer.IDENTIFIER,
            token.getChannel(),
            token.getStartIndex(),
            token.getStopIndex()));
}
 

import org.antlr.v4.runtime.CommonToken; //导入依赖的package包/类
/**
 * Cloning expression to create new same expression.
 */
public static ExprContext cloneExprContext(final ExprContext expr) {

  final ExprContext clone = createContextType(expr);

  clone.copyFrom(expr);

  for (final ParseTree child : expr.children) {
    if (child instanceof TerminalNode) {
      clone.addChild(new TerminalNodeImpl(((TerminalNode) child).getSymbol()));
    } else if (child instanceof ExprContext) {
      final ExprContext cloneChild = cloneExprContext((ExprContext) child);
      clone.addChild(cloneChild);
      setLeftRight(clone, cloneChild);
    } else if (child instanceof Token) {
      clone.addChild(new CommonToken((Token) child));
    }
  }
  return clone;
}
 

import org.antlr.v4.runtime.CommonToken; //导入依赖的package包/类
/**
 * Creating conjunction expression.
 */
public static ConjunctionContext createConjunctionContext(final ExprContext leftContext,
    final ExprContext rightContext) {

  final ConjunctionContext conjunctionContext = new ConjunctionContext(new ExprContext());
  final TerminalNodeImpl andNode = new TerminalNodeImpl(new CommonToken(10, "and"));

  // Setting context parents.
  leftContext.parent = conjunctionContext;
  andNode.parent = conjunctionContext;
  rightContext.parent = conjunctionContext;

  conjunctionContext.left = leftContext;
  conjunctionContext.right = rightContext;

  // Adding conjunction expression's children.
  conjunctionContext.addChild(leftContext);
  conjunctionContext.addChild(andNode);
  conjunctionContext.addChild(rightContext);

  return conjunctionContext;
}
 

import org.antlr.v4.runtime.CommonToken; //导入依赖的package包/类
/**
 * Creating disjunction expression.
 */
public static DisjunctionContext createDisjunctionContext(final ExprContext leftContext,
    final ExprContext rightContext) {

  final DisjunctionContext disjunctionContext = new DisjunctionContext(new ExprContext());
  final TerminalNodeImpl orNode = new TerminalNodeImpl(new CommonToken(12, "or"));

  // Setting context parents.
  leftContext.parent = disjunctionContext;
  rightContext.parent = disjunctionContext;
  orNode.parent = disjunctionContext;

  disjunctionContext.left = leftContext;
  disjunctionContext.right = rightContext;

  // Adding disjunction expression's children.
  disjunctionContext.addChild(leftContext);
  disjunctionContext.addChild(orNode);
  disjunctionContext.addChild(rightContext);

  return disjunctionContext;
}
 

import org.antlr.v4.runtime.CommonToken; //导入依赖的package包/类
/**
 * Creating negation expression.
 */
public static NegationContext createNegationContext(final ExprContext expr) {

  final NegationContext negationContext = new NegationContext(new ExprContext());
  final TerminalNodeImpl notNode = new TerminalNodeImpl(new CommonToken(FOLParser.NOT, "not"));

  // Setting context parents.
  notNode.parent = negationContext;
  expr.parent = negationContext;

  // Adding negation expression's children.
  negationContext.addChild(notNode);
  negationContext.addChild(expr);

  return negationContext;
}
 

import org.antlr.v4.runtime.CommonToken; //导入依赖的package包/类
/**
 * Creating parentheses expression.
 */
public static ParenthesesContext createParenthesesContext(final ExprContext expr) {

  final ParenthesesContext parenthesesContext = new ParenthesesContext(new ExprContext());
  final TerminalNodeImpl leftParenthes = new TerminalNodeImpl(new CommonToken(FOLParser.LP, "("));
  final TerminalNodeImpl rightParenthes =
      new TerminalNodeImpl(new CommonToken(FOLParser.RP, ")"));

  // Setting context parents.
  leftParenthes.parent = parenthesesContext;
  rightParenthes.parent = parenthesesContext;
  expr.parent = parenthesesContext;

  // Adding parentheses expression's children.
  parenthesesContext.addChild(leftParenthes);
  parenthesesContext.addChild(expr);
  parenthesesContext.addChild(rightParenthes);

  return parenthesesContext;
}
 

import org.antlr.v4.runtime.CommonToken; //导入依赖的package包/类
@Test
public void evaluate_returns_result_for_valid_CEF_string() throws Exception {
    final Map<String, Expression> arguments = ImmutableMap.of(
            CEFParserFunction.VALUE, new StringExpression(new CommonToken(0), "CEF:0|vendor|product|1.0|id|name|low|dvc=example.com msg=Foobar"),
            CEFParserFunction.USE_FULL_NAMES, new BooleanExpression(new CommonToken(0), false)
    );
    final FunctionArgs functionArgs = new FunctionArgs(function, arguments);
    final Message message = new Message("__dummy", "__dummy", DateTime.parse("2010-07-30T16:03:25Z"));
    final EvaluationContext evaluationContext = new EvaluationContext(message);

    final CEFParserResult result = function.evaluate(functionArgs, evaluationContext);
    assertNotNull(result);
    assertEquals(0, result.get("cef_version"));
    assertEquals("vendor", result.get("device_vendor"));
    assertEquals("product", result.get("device_product"));
    assertEquals("1.0", result.get("device_version"));
    assertEquals("id", result.get("device_event_class_id"));
    assertEquals("low", result.get("severity"));
    assertEquals("example.com", result.get("dvc"));
    assertEquals("Foobar", result.get("msg"));
}
 

import org.antlr.v4.runtime.CommonToken; //导入依赖的package包/类
@Test
public void evaluate_returns_result_for_valid_CEF_string_with_short_names_if_useFullNames_parameter_is_missing() throws Exception {
    final Map<String, Expression> arguments = Collections.singletonMap(
            CEFParserFunction.VALUE, new StringExpression(new CommonToken(0), "CEF:0|vendor|product|1.0|id|name|low|dvc=example.com msg=Foobar")
    );
    final FunctionArgs functionArgs = new FunctionArgs(function, arguments);
    final Message message = new Message("__dummy", "__dummy", DateTime.parse("2010-07-30T16:03:25Z"));
    final EvaluationContext evaluationContext = new EvaluationContext(message);

    final CEFParserResult result = function.evaluate(functionArgs, evaluationContext);
    assertNotNull(result);
    assertEquals(0, result.get("cef_version"));
    assertEquals("vendor", result.get("device_vendor"));
    assertEquals("product", result.get("device_product"));
    assertEquals("1.0", result.get("device_version"));
    assertEquals("id", result.get("device_event_class_id"));
    assertEquals("low", result.get("severity"));
    assertEquals("example.com", result.get("dvc"));
    assertEquals("Foobar", result.get("msg"));
}
 

import org.antlr.v4.runtime.CommonToken; //导入依赖的package包/类
@Test
public void evaluate_returns_result_for_valid_CEF_string_with_full_names() throws Exception {
    final CEFParserFunction function = new CEFParserFunction(new MetricRegistry());
    final Map<String, Expression> arguments = ImmutableMap.of(
            CEFParserFunction.VALUE, new StringExpression(new CommonToken(0), "CEF:0|vendor|product|1.0|id|name|low|dvc=example.com msg=Foobar"),
            CEFParserFunction.USE_FULL_NAMES, new BooleanExpression(new CommonToken(0), true)
    );
    final FunctionArgs functionArgs = new FunctionArgs(function, arguments);
    final Message message = new Message("__dummy", "__dummy", DateTime.parse("2010-07-30T16:03:25Z"));
    final EvaluationContext evaluationContext = new EvaluationContext(message);

    final CEFParserResult result = function.evaluate(functionArgs, evaluationContext);
    assertNotNull(result);
    assertEquals(0, result.get("cef_version"));
    assertEquals("vendor", result.get("device_vendor"));
    assertEquals("product", result.get("device_product"));
    assertEquals("1.0", result.get("device_version"));
    assertEquals("id", result.get("device_event_class_id"));
    assertEquals("low", result.get("severity"));
    assertEquals("example.com", result.get("deviceAddress"));
    assertEquals("Foobar", result.get("message"));
}
 

import org.antlr.v4.runtime.CommonToken; //导入依赖的package包/类
@Test
public void evaluate_returns_result_without_message_field() throws Exception {
    final Map<String, Expression> arguments = ImmutableMap.of(
            CEFParserFunction.VALUE, new StringExpression(new CommonToken(0), "CEF:0|vendor|product|1.0|id|name|low|dvc=example.com"),
            CEFParserFunction.USE_FULL_NAMES, new BooleanExpression(new CommonToken(0), false)
    );
    final FunctionArgs functionArgs = new FunctionArgs(function, arguments);
    final Message message = new Message("__dummy", "__dummy", DateTime.parse("2010-07-30T16:03:25Z"));
    final EvaluationContext evaluationContext = new EvaluationContext(message);

    final CEFParserResult result = function.evaluate(functionArgs, evaluationContext);
    assertNotNull(result);
    assertEquals(0, result.get("cef_version"));
    assertEquals("vendor", result.get("device_vendor"));
    assertEquals("product", result.get("device_product"));
    assertEquals("1.0", result.get("device_version"));
    assertEquals("id", result.get("device_event_class_id"));
    assertEquals("low", result.get("severity"));
    assertEquals("example.com", result.get("dvc"));
    assertFalse(result.containsKey("message"));
}
 

import org.antlr.v4.runtime.CommonToken; //导入依赖的package包/类
public static void wipeCharPositionInfoAndWhitespaceTokens(CodeBuffTokenStream tokens) {
	tokens.fill();
	CommonToken dummy = new CommonToken(Token.INVALID_TYPE, "");
	dummy.setChannel(Token.HIDDEN_CHANNEL);
	Token firstRealToken = tokens.getNextRealToken(-1);
	for (int i = 0; i<tokens.size(); i++) {
		if ( i==firstRealToken.getTokenIndex() ) continue; // don't wack first token
		CommonToken t = (CommonToken)tokens.get(i);
		if ( t.getText().matches("\\s+") ) {
			tokens.getTokens().set(i, dummy); // wack whitespace token so we can't use it during prediction
		}
		else {
			t.setLine(0);
			t.setCharPositionInLine(-1);
		}
	}
}
 

import org.antlr.v4.runtime.CommonToken; //导入依赖的package包/类
public MplProgram assemble(File programFile) throws IOException {
  programFile = getCanonicalFile(programFile);
  // Don't cache the first interpreter, because it's program is mutable and will be changed
  MplInterpreter main = MplInterpreter.interpret(programFile, context);
  MplProgram program = main.getProgram();
  programBuilder = new MplProgramBuilder(program, programFile);
  resolveReferences(main.getReferences().values());
  if (!program.isScript()) {
    doIncludes();
  }
  MplProgram result = programBuilder.getProgram();
  boolean containsRemoteProcess = result.getProcesses().stream()//
      .anyMatch(p -> p.getType() == ProcessType.REMOTE);
  if (context.getErrors().isEmpty() && !containsRemoteProcess) {
    context.addError(
        new CompilerException(new MplSource(programFile, "", new CommonToken(MplLexer.PROCESS)),
            "This file does not include any remote processes"));
  }
  return result;
}
 

import org.antlr.v4.runtime.CommonToken; //导入依赖的package包/类
@Override
	public void syntaxError(Recognizer<?, ?> recognizer,
							Object offendingSymbol,
							int line,
							int charPositionInLine,
							String msg,
							RecognitionException e)
	{
		if ( offendingSymbol==null ) {
			final Lexer lexer = (Lexer) recognizer;
			int i = lexer.getCharIndex();
			final int n = lexer.getInputStream().size();
			if (i >= n) {
				i = n - 1;
			}
			final String text = lexer.getInputStream().getText(new Interval(i, i));
			CommonToken t = (CommonToken) lexer.getTokenFactory().create(Token.INVALID_TYPE, text);
			t.setStartIndex(i);
			t.setStopIndex(i);
			t.setLine(line);
			t.setCharPositionInLine(charPositionInLine);
			offendingSymbol = t;
		}
//		System.out.println("lex error: " + offendingSymbol);
		issues.add(new Issue(msg, (Token)offendingSymbol));
	}
 

import org.antlr.v4.runtime.CommonToken; //导入依赖的package包/类
/**
 * @see BaseErrorListener#reportAmbiguity
 */
@Override
public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line,
    int charPositionInLine, String msg, RecognitionException e) {
    List<String> stack = ((Parser) recognizer).getRuleInvocationStack();
    Collections.reverse(stack);
    String logMsg = "Parser ERROR: line " + line + ":" + charPositionInLine + " at "
        + offendingSymbol + ": " + msg;
    CommonToken tok = (CommonToken) offendingSymbol;
    String s = tok.getText();
    logMsg += ": offending token " + s;
    if (s.equals("<EOF>")) {
        logMsg += ". Look for tag=(null or empty).";
    } else {
        try {
            Integer.parseInt(s);
        } catch (NumberFormatException ex) {
            logMsg += " not a number. ";
        }
    }
    FixRulesParserErrorListener.logger.error(logMsg + " Tree = {}", stack);
    throw new RuntimeException(logMsg);
}
 

import org.antlr.v4.runtime.CommonToken; //导入依赖的package包/类
@Test
@Ignore
public void testAllRpgSources() throws Exception {
    List<String> errors = new ArrayList<String>();
    List<String> files = new ArrayList<String>();
    //File dir = new File(getResourcePath("./"));
    File dir = new File("c:\\temp\\rpg\\all");

    int count=0;
    for (File file : dir.listFiles()) {
        if (isRpgSourceFile(file)) {
            String rpgsource = TestUtils.loadFile(file);
            rpgsource = TestUtils.padSourceLines(rpgsource, false);
            List<CommonToken> tokenList = TestUtils.getParsedTokens(rpgsource, errors);
            if (errors.size() > 0) {
            		System.out.println("The failing file is :"  + file.getName());
            	if(count++>10)
                break;
            	errors.clear();
            }
        }
    }
    assertThat(errors, is(empty()));
}