Java Parser.addParseTarget方法代码示例

import nl.basjes.parse.core.Parser; //导入方法依赖的package包/类
public static Parser<TestRecord> createTestParser() throws NoSuchMethodException {
    Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, getLogFormat());

    parser.addDissector(new nl.basjes.parse.httpdlog.dissectors.ScreenResolutionDissector());

    parser.addTypeRemapping("request.firstline.uri.query.g", "HTTP.URI");
    parser.addTypeRemapping("request.firstline.uri.query.r", "HTTP.URI");
    parser.addTypeRemapping("request.firstline.uri.query.s", "SCREENRESOLUTION");

    parser.addParseTarget("setConnectionClientHost", "IP:connection.client.host");
    parser.addParseTarget("setRequestReceiveTime",   "TIME.STAMP:request.receive.time");
    parser.addParseTarget("setReferrer",             "STRING:request.firstline.uri.query.g.query.promo");
    parser.addParseTarget("setScreenResolution",     "STRING:request.firstline.uri.query.s");
    parser.addParseTarget("setScreenWidth",          "SCREENWIDTH:request.firstline.uri.query.s.width");
    parser.addParseTarget("setScreenHeight",         "SCREENHEIGHT:request.firstline.uri.query.s.height");
    parser.addParseTarget("setGoogleQuery",          "STRING:request.firstline.uri.query.r.query.blabla");
    parser.addParseTarget("setBui",                  "HTTP.COOKIE:request.cookies.bui");
    parser.addParseTarget("setUseragent",            "HTTP.USERAGENT:request.user-agent");
    return parser;
}
 

import nl.basjes.parse.core.Parser; //导入方法依赖的package包/类
private void printAllPossibles(String logformat) throws NoSuchMethodException, MissingDissectorsException, InvalidDissectorException {
    // To figure out what values we CAN get from this line we instantiate the parser with a dummy class
    // that does not have ANY @Field annotations.
    Parser<Object> dummyParser= new HttpdLoglineParser<>(Object.class, logformat);

    List<String> possiblePaths;
    possiblePaths = dummyParser.getPossiblePaths();

    // If you want to call 'getCasts' then the actual parser needs to be constructed.
    // Simply calling getPossiblePaths does not build the actual parser.
    // Because we want this for all possibilities yet we are never actually going to use this instance of the parser
    // We simply give it a random method with the right signature and tell it we want all possible paths
    dummyParser.addParseTarget(String.class.getMethod("indexOf", String.class), possiblePaths);

    LOG.info("==================================");
    LOG.info("Possible output:");
    for (String path : possiblePaths) {
        LOG.info("{}     {}", path, dummyParser.getCasts(path));
    }
    LOG.info("==================================");
}
 

import nl.basjes.parse.core.Parser; //导入方法依赖的package包/类
public static Parser<TestRecord> createTestParser() throws NoSuchMethodException {
    Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, getLogFormat());

    parser.addDissector(new ScreenResolutionDissector());

    parser.addTypeRemapping("request.firstline.uri.query.g", "HTTP.URI");
    parser.addTypeRemapping("request.firstline.uri.query.r", "HTTP.URI");
    parser.addTypeRemapping("request.firstline.uri.query.s", "SCREENRESOLUTION");

    parser.addParseTarget("setConnectionClientHost", "IP:connection.client.host");
    parser.addParseTarget("setRequestReceiveTime",   "TIME.STAMP:request.receive.time");
    parser.addParseTarget("setReferrer",             "STRING:request.firstline.uri.query.g.query.promo");
    parser.addParseTarget("setScreenResolution",     "STRING:request.firstline.uri.query.s");
    parser.addParseTarget("setScreenWidth",          "SCREENWIDTH:request.firstline.uri.query.s.width");
    parser.addParseTarget("setScreenHeight",         "SCREENHEIGHT:request.firstline.uri.query.s.height");
    parser.addParseTarget("setGoogleQuery",          "STRING:request.firstline.uri.query.r.query.blabla");
    parser.addParseTarget("setBui",                  "HTTP.COOKIE:request.cookies.bui");
    parser.addParseTarget("setUseragent",            "HTTP.USERAGENT:request.user-agent");
    return parser;
}
 

import nl.basjes.parse.core.Parser; //导入方法依赖的package包/类
/**
 * This record will be used with a single parser. For each field that is to be parsed a setter will be called. It
 * registers a setter method for each field being parsed. It also builds the data writers to hold the data beings
 * parsed.
 *
 * @param parser
 * @param mapWriter
 * @param type
 * @param parserFieldName
 * @param drillFieldName
 * @throws NoSuchMethodException
 */
public void addField(final Parser<HttpdLogRecord> parser, final MapWriter mapWriter, final EnumSet<Casts> type, final String parserFieldName, final String drillFieldName) throws NoSuchMethodException {
  final boolean hasWildcard = parserFieldName.endsWith(HttpdParser.PARSER_WILDCARD);

  /**
   * This is a dynamic way to map the setter for each specified field type. <br/>
   * e.g. a TIME.STAMP may map to a LONG while a referrer may map to a STRING
   */
  if (hasWildcard) {
    final String cleanName = parserFieldName.substring(0, parserFieldName.length() - HttpdParser.PARSER_WILDCARD.length());
    LOG.debug("Adding WILDCARD parse target: {} as {}, with field name: {}", parserFieldName, cleanName, drillFieldName);
    parser.addParseTarget(this.getClass().getMethod("setWildcard", String.class, String.class), parserFieldName);
    parser.addParseTarget(this.getClass().getMethod("setWildcard", String.class, Double.class), parserFieldName);
    parser.addParseTarget(this.getClass().getMethod("setWildcard", String.class, Long.class), parserFieldName);
    wildcards.put(cleanName, mapWriter.map(drillFieldName));
  }
  else if (type.contains(Casts.DOUBLE)) {
    LOG.debug("Adding DOUBLE parse target: {}, with field name: {}", parserFieldName, drillFieldName);
    parser.addParseTarget(this.getClass().getMethod("set", String.class, Double.class), parserFieldName);
    doubles.put(parserFieldName, mapWriter.float8(drillFieldName));
  }
  else if (type.contains(Casts.LONG)) {
    LOG.debug("Adding LONG parse target: {}, with field name: {}", parserFieldName, drillFieldName);
    parser.addParseTarget(this.getClass().getMethod("set", String.class, Long.class), parserFieldName);
    longs.put(parserFieldName, mapWriter.bigInt(drillFieldName));
  }
  else {
    LOG.debug("Adding STRING parse target: {}, with field name: {}", parserFieldName, drillFieldName);
    parser.addParseTarget(this.getClass().getMethod("set", String.class, String.class), parserFieldName);
    strings.put(parserFieldName, mapWriter.varChar(drillFieldName));
  }
}
 

import nl.basjes.parse.core.Parser; //导入方法依赖的package包/类
@Test
public void testBasicParsing() throws Exception {
    Parser<MyRecord> parser = new HttpdLoglineParser<>(MyRecord.class, LOG_FORMAT);
    MyRecord         record = new MyRecord();

    List<String> paths = parser.getPossiblePaths();

    parser.addParseTarget(record.getClass().getMethod("setValue", String.class, String.class), paths);

    for (String logline : LOG_LINES) {
        record.clear();
        parser.parse(record, logline);
        System.out.println(record.toString());
    }
}
 

import nl.basjes.parse.core.Parser; //导入方法依赖的package包/类
private Parser<ParsedRecord> createParser() throws IOException {
    if (fieldList == null || logformat == null) {
        return null;
    }

    Parser<ParsedRecord> newParser;
    try {
        newParser = instantiateParser(logformat);

        for (String field: fieldList) {
            if (field.endsWith(".*")) {
                newParser.addParseTarget(ParsedRecord.class.getMethod("setMultiValueString",
                        String.class, String.class), field);
            } else {
                newParser.addParseTarget(ParsedRecord.class.getMethod("set",
                        String.class, String.class), field);
                newParser.addParseTarget(ParsedRecord.class.getMethod("set",
                        String.class, Long.class), field);
                newParser.addParseTarget(ParsedRecord.class.getMethod("set",
                        String.class, Double.class), field);
            }
        }

    } catch (NoSuchMethodException
            |SecurityException e) {
        throw new IOException(e.toString());
    }
    return newParser;
}
 

import nl.basjes.parse.core.Parser; //导入方法依赖的package包/类
private void setupParser(final MapWriter mapWriter, final String logFormat, final Map<String, String> fieldMapping)
    throws NoSuchMethodException, MissingDissectorsException, InvalidDissectorException {

  /**
   * If the user has selected fields, then we will use them to configure the parser because this would be the most
   * efficient way to parse the log.
   */
  final Map<String, String> requestedPaths;
  final List<String> allParserPaths = parser.getPossiblePaths();
  if (fieldMapping != null && !fieldMapping.isEmpty()) {
    LOG.debug("Using fields defined by user.");
    requestedPaths = fieldMapping;
  }
  else {
    /**
     * Use all possible paths that the parser has determined from the specified log format.
     */
    LOG.debug("No fields defined by user, defaulting to all possible fields.");
    requestedPaths = Maps.newHashMap();
    for (final String parserPath : allParserPaths) {
      requestedPaths.put(drillFormattedFieldName(parserPath), parserPath);
    }
  }

  /**
   * By adding the parse target to the dummy instance we activate it for use. Which we can then use to find out which
   * paths cast to which native data types. After we are done figuring this information out, we throw this away
   * because this will be the slowest parsing path possible for the specified format.
   */
  Parser<Object> dummy = new HttpdLoglineParser<>(Object.class, logFormat);
  dummy.addParseTarget(String.class.getMethod("indexOf", String.class), allParserPaths);

  for (final Map.Entry<String, String> entry : requestedPaths.entrySet()) {
    final EnumSet<Casts> casts;

    /**
     * Check the field specified by the user to see if it is supposed to be remapped.
     */
    if (entry.getValue().startsWith(REMAPPING_FLAG)) {
      /**
       * Because this field is being remapped we need to replace the field name that the parser uses.
       */
      entry.setValue(entry.getValue().substring(REMAPPING_FLAG.length()));

      final String[] pieces = entry.getValue().split(":");
      addTypeRemapping(parser, pieces[1], pieces[0]);

      casts = Casts.STRING_ONLY;
    }
    else {
      casts = dummy.getCasts(entry.getValue());
    }

    LOG.debug("Setting up drill field: {}, parser field: {}, which casts as: {}", entry.getKey(), entry.getValue(), casts);
    record.addField(parser, mapWriter, casts, entry.getValue(), entry.getKey());
  }
}
 

import nl.basjes.parse.core.Parser; //导入方法依赖的package包/类
/**
 * Test of initialize method, of class ApacheHttpdLogParser.
 */
@Test
public void fullTest1() throws Exception {
    String line = "%127.0.0.1 127.0.0.1 127.0.0.1 - - [31/Dec/2012:23:49:40 +0100] "
            + "\"GET /icons/powered_by_rh.png?aap=noot&res=1024x768 HTTP/1.1\" 200 1213 "
            + "80 \"\" \"http://localhost/index.php?mies=wim\" 351 "
            + "\"Mozilla/5.0 (X11; Linux i686 on x86_64; rv:11.0) Gecko/20100101 Firefox/11.0\" "
            + "\"jquery-ui-theme=Eggplant\" \"Apache=127.0.0.1.1344635380111339; path=/; domain=.basjes.nl\" \"-\" "
            + "\"\\\"3780ff-4bd-4c1ce3df91380\\\"\"";

    Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, LOG_FORMAT);

    // Manually add an extra dissector
    parser.addDissector(new ScreenResolutionDissector());
    parser.addTypeRemapping("request.firstline.uri.query.res", "SCREENRESOLUTION");
    List<String> extraFields = new ArrayList<>();
    extraFields.add("SCREENWIDTH:request.firstline.uri.query.res.width");
    extraFields.add("SCREENHEIGHT:request.firstline.uri.query.res.height");
    parser.addParseTarget(TestRecord.class.getMethod("setValue", String.class, String.class), extraFields);

    TestRecord record = new TestRecord();
    parser.parse(record, line);
    Map<String, String> results = record.getResults();

    System.out.println(results.toString());

    assertEquals("noot", results.get("STRING:request.firstline.uri.query.aap"));
    assertEquals(null, results.get("STRING:request.firstline.uri.query.foo"));
    assertEquals(null, results.get("STRING:request.querystring.aap"));
    assertEquals("1024", results.get("SCREENWIDTH:request.firstline.uri.query.res.width"));
    assertEquals("768", results.get("SCREENHEIGHT:request.firstline.uri.query.res.height"));

    assertEquals("127.0.0.1", results.get("IP:connection.client.ip"));
    assertEquals(null, results.get("NUMBER:connection.client.logname"));
    assertEquals(null, results.get("STRING:connection.client.user"));
    assertEquals("31/Dec/2012:23:49:40 +0100", results.get("TIME.STAMP:request.receive.time"));
    assertEquals("1356994180000", results.get("TIME.EPOCH:request.receive.time.epoch"));
    assertEquals("1", results.get("TIME.WEEK:request.receive.time.weekofweekyear"));
    assertEquals("2013", results.get("TIME.YEAR:request.receive.time.weekyear"));
    assertEquals("2012", results.get("TIME.YEAR:request.receive.time.year"));
    assertEquals("40", results.get("TIME.SECOND:request.receive.time.second"));
    assertEquals("/icons/powered_by_rh.png?aap=noot&res=1024x768", results.get("HTTP.URI:request.firstline.uri"));
    assertEquals("200", results.get("STRING:request.status.last"));
    assertEquals("1213", results.get("BYTESCLF:response.body.bytes"));
    assertEquals("http://localhost/index.php?mies=wim", results.get("HTTP.URI:request.referer"));
    assertEquals("wim", results.get("STRING:request.referer.query.mies"));
    assertEquals("Mozilla/5.0 (X11; Linux i686 on x86_64; rv:11.0) Gecko/20100101 Firefox/11.0",
            results.get("HTTP.USERAGENT:request.user-agent"));
    assertEquals("31", results.get("TIME.DAY:request.receive.time.day"));
    assertEquals("23", results.get("TIME.HOUR:request.receive.time.hour"));
    assertEquals("December", results.get("TIME.MONTHNAME:request.receive.time.monthname"));
    assertEquals("351", results.get("MICROSECONDS:response.server.processing.time"));
    assertEquals("Apache=127.0.0.1.1344635380111339; path=/; domain=.basjes.nl",
            results.get("HTTP.SETCOOKIES:response.cookies"));
    assertEquals("jquery-ui-theme=Eggplant", results.get("HTTP.COOKIES:request.cookies"));
    assertEquals("\\\"3780ff-4bd-4c1ce3df91380\\\"", results.get("HTTP.HEADER:response.header.etag"));

    assertEquals("Eggplant", results.get("HTTP.COOKIE:request.cookies.jquery-ui-theme"));
    assertEquals("Apache=127.0.0.1.1344635380111339; path=/; domain=.basjes.nl", results.get("HTTP.SETCOOKIE:response.cookies.apache"));
    assertEquals(".basjes.nl", results.get("STRING:response.cookies.apache.domain"));

}