Java Parser类代码示例

import nl.basjes.parse.core.Parser; //导入依赖的package包/类
public static Parser<TestRecord> createTestParser() throws NoSuchMethodException {
    Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, getLogFormat());

    parser.addDissector(new nl.basjes.parse.httpdlog.dissectors.ScreenResolutionDissector());

    parser.addTypeRemapping("request.firstline.uri.query.g", "HTTP.URI");
    parser.addTypeRemapping("request.firstline.uri.query.r", "HTTP.URI");
    parser.addTypeRemapping("request.firstline.uri.query.s", "SCREENRESOLUTION");

    parser.addParseTarget("setConnectionClientHost", "IP:connection.client.host");
    parser.addParseTarget("setRequestReceiveTime",   "TIME.STAMP:request.receive.time");
    parser.addParseTarget("setReferrer",             "STRING:request.firstline.uri.query.g.query.promo");
    parser.addParseTarget("setScreenResolution",     "STRING:request.firstline.uri.query.s");
    parser.addParseTarget("setScreenWidth",          "SCREENWIDTH:request.firstline.uri.query.s.width");
    parser.addParseTarget("setScreenHeight",         "SCREENHEIGHT:request.firstline.uri.query.s.height");
    parser.addParseTarget("setGoogleQuery",          "STRING:request.firstline.uri.query.r.query.blabla");
    parser.addParseTarget("setBui",                  "HTTP.COOKIE:request.cookies.bui");
    parser.addParseTarget("setUseragent",            "HTTP.USERAGENT:request.user-agent");
    return parser;
}
 

import nl.basjes.parse.core.Parser; //导入依赖的package包/类
private void printAllPossibles(String logformat) throws NoSuchMethodException, MissingDissectorsException, InvalidDissectorException {
    // To figure out what values we CAN get from this line we instantiate the parser with a dummy class
    // that does not have ANY @Field annotations.
    Parser<Object> dummyParser= new HttpdLoglineParser<>(Object.class, logformat);

    List<String> possiblePaths;
    possiblePaths = dummyParser.getPossiblePaths();

    // If you want to call 'getCasts' then the actual parser needs to be constructed.
    // Simply calling getPossiblePaths does not build the actual parser.
    // Because we want this for all possibilities yet we are never actually going to use this instance of the parser
    // We simply give it a random method with the right signature and tell it we want all possible paths
    dummyParser.addParseTarget(String.class.getMethod("indexOf", String.class), possiblePaths);

    LOG.info("==================================");
    LOG.info("Possible output:");
    for (String path : possiblePaths) {
        LOG.info("{}     {}", path, dummyParser.getCasts(path));
    }
    LOG.info("==================================");
}
 

import nl.basjes.parse.core.Parser; //导入依赖的package包/类
public static Parser<TestRecord> createTestParser() throws NoSuchMethodException {
    Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, getLogFormat());

    parser.addDissector(new ScreenResolutionDissector());

    parser.addTypeRemapping("request.firstline.uri.query.g", "HTTP.URI");
    parser.addTypeRemapping("request.firstline.uri.query.r", "HTTP.URI");
    parser.addTypeRemapping("request.firstline.uri.query.s", "SCREENRESOLUTION");

    parser.addParseTarget("setConnectionClientHost", "IP:connection.client.host");
    parser.addParseTarget("setRequestReceiveTime",   "TIME.STAMP:request.receive.time");
    parser.addParseTarget("setReferrer",             "STRING:request.firstline.uri.query.g.query.promo");
    parser.addParseTarget("setScreenResolution",     "STRING:request.firstline.uri.query.s");
    parser.addParseTarget("setScreenWidth",          "SCREENWIDTH:request.firstline.uri.query.s.width");
    parser.addParseTarget("setScreenHeight",         "SCREENHEIGHT:request.firstline.uri.query.s.height");
    parser.addParseTarget("setGoogleQuery",          "STRING:request.firstline.uri.query.r.query.blabla");
    parser.addParseTarget("setBui",                  "HTTP.COOKIE:request.cookies.bui");
    parser.addParseTarget("setUseragent",            "HTTP.USERAGENT:request.user-agent");
    return parser;
}
 

import nl.basjes.parse.core.Parser; //导入依赖的package包/类
/**
 * Test of initialize method, of class ApacheHttpdLogParser.
 */
@Test
public void fullTest1() throws Exception {

    String logFormat = LOG_FORMAT_1 + '\n'
                     + '\n'
                     + LOG_FORMAT_2 + '\n'
                     + '\n';

    Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, logFormat);

    validateLine1(parser);
    validateLine1(parser);
    validateLine2(parser);
    validateLine2(parser);
    validateLine1(parser);
    validateLine1(parser);
    validateLine2(parser);
    validateLine2(parser);
    validateLine1(parser);
    validateLine1(parser);
    validateLine2(parser);
    validateLine2(parser);
}
 

import nl.basjes.parse.core.Parser; //导入依赖的package包/类
private void setupFields() throws MissingDissectorsException, InvalidDissectorException, NoSuchMethodException, IOException {
    if (fieldList == null || fieldList.isEmpty()) {
        return; // Nothing to do here
    }
    String firstField = fieldList.get(0);
    if (fieldList.size() == 1 &&
        firstField.toLowerCase().trim().equals(FIELDS)) {
        outputAllPossibleFields = true;
        allPossiblePaths = getParser().getPossiblePaths();
        allPossiblePathsFieldName = firstField;
        Parser<ParsedRecord> newParser = instantiateParser(logformat)
            .addParseTarget(ParsedRecord.class.getMethod("set", String.class, String.class), allPossiblePaths)
            .addTypeRemappings(typeRemappings);
        allCasts = newParser.getAllCasts();
    }
}
 

import nl.basjes.parse.core.Parser; //导入依赖的package包/类
/**
 * This record will be used with a single parser. For each field that is to be parsed a setter will be called. It
 * registers a setter method for each field being parsed. It also builds the data writers to hold the data beings
 * parsed.
 *
 * @param parser
 * @param mapWriter
 * @param type
 * @param parserFieldName
 * @param drillFieldName
 * @throws NoSuchMethodException
 */
public void addField(final Parser<HttpdLogRecord> parser, final MapWriter mapWriter, final EnumSet<Casts> type, final String parserFieldName, final String drillFieldName) throws NoSuchMethodException {
  final boolean hasWildcard = parserFieldName.endsWith(HttpdParser.PARSER_WILDCARD);

  /**
   * This is a dynamic way to map the setter for each specified field type. <br/>
   * e.g. a TIME.STAMP may map to a LONG while a referrer may map to a STRING
   */
  if (hasWildcard) {
    final String cleanName = parserFieldName.substring(0, parserFieldName.length() - HttpdParser.PARSER_WILDCARD.length());
    LOG.debug("Adding WILDCARD parse target: {} as {}, with field name: {}", parserFieldName, cleanName, drillFieldName);
    parser.addParseTarget(this.getClass().getMethod("setWildcard", String.class, String.class), parserFieldName);
    parser.addParseTarget(this.getClass().getMethod("setWildcard", String.class, Double.class), parserFieldName);
    parser.addParseTarget(this.getClass().getMethod("setWildcard", String.class, Long.class), parserFieldName);
    wildcards.put(cleanName, mapWriter.map(drillFieldName));
  }
  else if (type.contains(Casts.DOUBLE)) {
    LOG.debug("Adding DOUBLE parse target: {}, with field name: {}", parserFieldName, drillFieldName);
    parser.addParseTarget(this.getClass().getMethod("set", String.class, Double.class), parserFieldName);
    doubles.put(parserFieldName, mapWriter.float8(drillFieldName));
  }
  else if (type.contains(Casts.LONG)) {
    LOG.debug("Adding LONG parse target: {}, with field name: {}", parserFieldName, drillFieldName);
    parser.addParseTarget(this.getClass().getMethod("set", String.class, Long.class), parserFieldName);
    longs.put(parserFieldName, mapWriter.bigInt(drillFieldName));
  }
  else {
    LOG.debug("Adding STRING parse target: {}, with field name: {}", parserFieldName, drillFieldName);
    parser.addParseTarget(this.getClass().getMethod("set", String.class, String.class), parserFieldName);
    strings.put(parserFieldName, mapWriter.varChar(drillFieldName));
  }
}
 

import nl.basjes.parse.core.Parser; //导入依赖的package包/类
@Test
public void testInlineDefinition() {
    List<String> logLines = Collections.singletonList(TestCase.getInputLine());

    // Apply Create, passing the list and the coder, to create the PCollection.
    PCollection<String> input = pipeline.apply(Create.of(logLines)).setCoder(StringUtf8Coder.of());

    PCollection<TestRecord> filledTestRecords = input
        .apply("Extract Elements from logline",
            ParDo.of(new DoFn<String, TestRecord>() {
                private Parser<TestRecord> parser;

                @Setup
                public void setup() throws NoSuchMethodException {
                    parser = TestCase.createTestParser();
                }

                @ProcessElement
                public void processElement(ProcessContext c) throws InvalidDissectorException, MissingDissectorsException, DissectionFailure {
                    c.output(parser.parse(c.element()));
                }
            }));

    TestRecord expected = new TestRecord().setFullValid();

    PAssert.that(filledTestRecords).containsInAnyOrder(expected);

    pipeline.run().waitUntilFinish();
}
 

import nl.basjes.parse.core.Parser; //导入依赖的package包/类
private void run() throws InvalidDissectorException, MissingDissectorsException, NoSuchMethodException, DissectionFailure {

        // This format and logline originate from here:
        // http://stackoverflow.com/questions/20349184/java-parse-log-file
        String logformat = "%t %u [%D %h %{True-Client-IP}i %{UNIQUE_ID}e %r] %{Cookie}i %s \"%{User-Agent}i\" \"%{host}i\" %l %b %{Referer}i";
        String logline = "[02/Dec/2013:14:10:30 -0000] - [52075 10.102.4.254 177.43.52.210 UpyU1gpmBAwAACfd5W0AAAAW GET /SS14-VTam-ny_019.j" +
                "pg.rendition.zoomable.jpg HTTP/1.1] hsfirstvisit=http%3A%2F%2Fwww.domain.com%2Fen-us||1372268254000; _opt_vi_3FNG8DZU=F870" +
                "DCFD-CBA4-4B6E-BB58-4605A78EE71A; __ptca=145721067.0aDxsZlIuM48.1372279055.1379945057.1379950362.9; __ptv_62vY4e=0aDxsZlIu" +
                "M48; __pti_62vY4e=0aDxsZlIuM48; __ptcz=145721067.1372279055.1.0.ptmcsr=(direct)|ptmcmd=(none)|ptmccn=(direct); __hstc=1457" +
                "21067.b86362bb7a1d257bfa2d1fb77e128a85.1372268254968.1379934256743.1379939561848.9; hubspotutk=b86362bb7a1d257bfa2d1fb77e1" +
                "28a85; USER_GROUP=julinho%3Afalse; has_js=1; WT_FPC=id=177.43.52.210-1491335248.30301337:lv=1385997780893:ss=1385997780893" +
                "; dtCookie=1F2E0E1037589799D8D503EB8CFA12A1|_default|1; RM=julinho%3A5248423ad3fe062f06c54915e6cde5cb45147977; wcid=UpyKsQ" +
                "pmBAwAABURyNoAAAAS%3A35d8227ba1e8a9a9cebaaf8d019a74777c32b4c8; Carte::KerberosLexicon_getWGSN=82ae3dcd1b956288c3c86bdbed6e" +
                "bcc0fd040e1e; UserData=Username%3AJULINHO%3AHomepage%3A1%3AReReg%3A0%3ATrialist%3A0%3ALanguage%3Aen%3ACcode%3Abr%3AForceRe" +
                "Reg%3A0; UserID=1356673%3A12345%3A1234567890%3A123%3Accode%3Abr; USER_DATA=1356673%3Ajulinho%3AJulio+Jose%3Ada+Silva%3Ajul" +
                "inho%40tecnoblu.com.br%3A0%3A1%3Aen%3Abr%3A%3AWGSN%3A1385990833.81925%3A82ae3dcd1b956288c3c86bdbed6ebcc0fd040e1e; MODE=FON" +
                "TIS; SECTION=%2Fcontent%2Fsection%2Fhome.html; edge_auth=ip%3D177.43.52.210~expires%3D1385994522~access%3D%2Fapps%2F%2A%21" +
                "%2Fbin%2F%2A%21%2Fcontent%2F%2A%21%2Fetc%2F%2A%21%2Fhome%2F%2A%21%2Flibs%2F%2A%21%2Freport%2F%2A%21%2Fsection%2F%2A%21%2Fw" +
                "gsn%2F%2A~md5%3D90e73ee10161c1afacab12c6ea30b4ef; __utma=94539802.1793276213.1372268248.1385572390.1385990581.16; __utmb=9" +
                "4539802.52.9.1385991739764; __utmc=94539802; __utmz=94539802.1372268248.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none);" +
                " WT_FPC=id=177.43.52.210-1491335248.30301337:lv=1386000374581:ss=1386000374581; dtPC=-; NSC_wtfswfs_xfcgbsn40-41=ffffffff0" +
                "96e1a1d45525d5f4f58455e445a4a423660; akamai-edge=5ac6e5b3d0bbe2ea771bb2916d8bab34ea222a6a 200 \"Mozilla/5.0 (Windows NT 6." +
                "2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.57 Safari/537.36\" \"www.domain.com\" - 463952 http://ww" +
                "w.domain.com/content/report/shows/New_York/KSHK/trip/s_s_14_ny_ww/sheers.html";

        printAllPossibles(logformat);

        Parser<MyRecord> parser = new HttpdLoglineParser<>(MyRecord.class, logformat);
        MyRecord record = new MyRecord();

        LOG.info("==================================================================================");
        parser.parse(record, logline);
        LOG.info(record.toString());
        LOG.info("==================================================================================");
    }
 

import nl.basjes.parse.core.Parser; //导入依赖的package包/类
@Test
public void testInlineDefinition() throws Exception {
    // set up the execution environment
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

    DataSet<String> input = env.fromElements(TestCase.getInputLine());

    DataSet<TestRecord> filledTestRecords = input
        .map(new RichMapFunction<String, TestRecord>() {
            private Parser<TestRecord> parser;

            @Override
            public void open(org.apache.flink.configuration.Configuration parameters) throws Exception {
                parser = TestCase.createTestParser();
            }

            @Override
            public TestRecord map(String line) throws Exception {
                return parser.parse(line);
            }
        }).name("Extract Elements from logline");

    filledTestRecords.print();

    List<TestRecord> result = filledTestRecords.collect();

    assertEquals(1, result.size());
    assertEquals(new TestRecord().setFullValid(), result.get(0));
}
 

import nl.basjes.parse.core.Parser; //导入依赖的package包/类
@Test
public void testInlineDefinitionAvro() throws Exception {
    // set up the execution environment
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

    DataSet<String> input = env.fromElements(TestCase.getInputLine());

    DataSet<Click> filledTestRecords = input
        .map(new RichMapFunction<String, Click>() {
            private Parser<ClickSetter> parser;

            @Override
            public void open(org.apache.flink.configuration.Configuration parameters) {
                parser = new HttpdLoglineParser<>(ClickSetter.class, TestCase.getLogFormat());

                parser.addDissector(new ScreenResolutionDissector());
                parser.addTypeRemapping("request.firstline.uri.query.s", "SCREENRESOLUTION");
            }

            @Override
            public Click map(String line) throws Exception {
                return parser.parse(line).build();
            }
        }).name("Extract Elements from logline");

    filledTestRecords.print();

    List<Click> result = filledTestRecords.collect();

    assertEquals(1, result.size());
    assertEquals(ExpectedClick.create(), result.get(0));
}
 

import nl.basjes.parse.core.Parser; //导入依赖的package包/类
@Override
public <RECORD> void createAdditionalDissectors(Parser<RECORD> parser) {
    super.createAdditionalDissectors(parser);
    parser.addDissector(new BinaryIPDissector());
    parser.addDissector(new ConvertSecondsWithMillisStringDissector("SECOND_MILLIS",            "MILLISECONDS"));
    parser.addDissector(new ConvertSecondsWithMillisStringDissector("TIME.EPOCH_SECOND_MILLIS", "TIME.EPOCH"));
    parser.addDissector(new ConvertMillisecondsIntoMicroseconds("MILLISECONDS", "MICROSECONDS"));
}
 

import nl.basjes.parse.core.Parser; //导入依赖的package包/类
@Test
public void fullTest2() throws Exception {
    Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, LOG_FORMAT);

    String line = "%127.0.0.1 127.0.0.1 127.0.0.1 - - [10/Aug/2012:23:55:11 +0200] \"GET /icons/powered_by_rh.png HTTP/1.1\" 200 1213 80"
            + " \"\" \"http://localhost/\" 1306 \"Mozilla/5.0 (X11; Linux i686 on x86_64; rv:11.0) Gecko/20100101 Firefox/11.0\""
            + " \"jquery-ui-theme=Eggplant; Apache=127.0.0.1.1344635667182858\" \"-\" \"-\" \"\\\"3780ff-4bd-4c1ce3df91380\\\"\"";

    TestRecord record = new TestRecord();
    parser.parse(record, line);
    Map<String, String> results = record.getResults();

    assertEquals(null, results.get("HTTP.QUERYSTRING:request.firstline.uri.query.foo"));
    assertEquals("127.0.0.1", results.get("IP:connection.client.ip"));
    assertEquals(null, results.get("NUMBER:connection.client.logname"));
    assertEquals(null, results.get("STRING:connection.client.user"));
    assertEquals("10/Aug/2012:23:55:11 +0200", results.get("TIME.STAMP:request.receive.time"));
    assertEquals("11", results.get("TIME.SECOND:request.receive.time.second"));
    assertEquals("/icons/powered_by_rh.png", results.get("HTTP.URI:request.firstline.uri"));
    assertEquals("200", results.get("STRING:request.status.last"));
    assertEquals("1213", results.get("BYTESCLF:response.body.bytes"));
    assertEquals("http://localhost/", results.get("HTTP.URI:request.referer"));
    assertEquals("Mozilla/5.0 (X11; Linux i686 on x86_64; rv:11.0) Gecko/20100101 Firefox/11.0",
            results.get("HTTP.USERAGENT:request.user-agent"));
    assertEquals("10", results.get("TIME.DAY:request.receive.time.day"));
    assertEquals("23", results.get("TIME.HOUR:request.receive.time.hour"));
    assertEquals("August", results.get("TIME.MONTHNAME:request.receive.time.monthname"));
    assertEquals("1306", results.get("MICROSECONDS:response.server.processing.time"));
    assertEquals(null, results.get("HTTP.SETCOOKIES:response.cookies"));
    assertEquals("jquery-ui-theme=Eggplant; Apache=127.0.0.1.1344635667182858",
            results.get("HTTP.COOKIES:request.cookies"));
    assertEquals("\\\"3780ff-4bd-4c1ce3df91380\\\"", results.get("HTTP.HEADER:response.header.etag"));
    // assertEquals("351",results.get("COOKIE:request.cookie.jquery-ui-theme"));
}
 

import nl.basjes.parse.core.Parser; //导入依赖的package包/类
@Test
public void fullTestTooLongUri() throws Exception {
    Parser<TestRecord> parser = new HttpdLoglineParser<>(TestRecord.class, LOG_FORMAT);

    String line = "%127.0.0.1 127.0.0.1 127.0.0.1 - - [10/Aug/2012:23:55:11 +0200] \"GET /ImagineAURLHereThatIsTooLong\" 414 1213 80"
            + " \"\" \"http://localhost/\" 1306 \"Mozilla/5.0 (X11; Linux i686 on x86_64; rv:11.0) Gecko/20100101 Firefox/11.0\""
            + " \"jquery-ui-theme=Eggplant; Apache=127.0.0.1.1344635667182858\" \"-\" \"-\" \"\\\"3780ff-4bd-4c1ce3df91380\\\"\"";

    TestRecord record = new TestRecord();
    parser.parse(record, line);
    Map<String, String> results = record.getResults();

    // System.out.println(results.toString());

    assertEquals(null, results.get("HTTP.QUERYSTRING:request.firstline.uri.query.foo"));
    assertEquals("127.0.0.1", results.get("IP:connection.client.ip"));
    assertEquals(null, results.get("NUMBER:connection.client.logname"));
    assertEquals(null, results.get("STRING:connection.client.user"));
    assertEquals("10/Aug/2012:23:55:11 +0200", results.get("TIME.STAMP:request.receive.time"));
    assertEquals("11", results.get("TIME.SECOND:request.receive.time.second"));
    assertEquals("/ImagineAURLHereThatIsTooLong", results.get("HTTP.URI:request.firstline.uri"));
    assertEquals("414", results.get("STRING:request.status.last"));
    assertEquals("1213", results.get("BYTESCLF:response.body.bytes"));
    assertEquals("http://localhost/", results.get("HTTP.URI:request.referer"));
    assertEquals("Mozilla/5.0 (X11; Linux i686 on x86_64; rv:11.0) Gecko/20100101 Firefox/11.0",
            results.get("HTTP.USERAGENT:request.user-agent"));
    assertEquals("10", results.get("TIME.DAY:request.receive.time.day"));
    assertEquals("23", results.get("TIME.HOUR:request.receive.time.hour"));
    assertEquals("August", results.get("TIME.MONTHNAME:request.receive.time.monthname"));
    assertEquals("1306", results.get("MICROSECONDS:response.server.processing.time"));
    assertEquals(null, results.get("HTTP.SETCOOKIES:response.cookies"));
    assertEquals("jquery-ui-theme=Eggplant; Apache=127.0.0.1.1344635667182858",
            results.get("HTTP.COOKIES:request.cookies"));
    assertEquals("\\\"3780ff-4bd-4c1ce3df91380\\\"", results.get("HTTP.HEADER:response.header.etag"));
    // assertEquals("351",results.get("COOKIE:request.cookie.jquery-ui-theme"));
}
 

import nl.basjes.parse.core.Parser; //导入依赖的package包/类
@Test
public void testMissing() throws Exception {
    try {
        Parser<TestRecordMissing> parser = new HttpdLoglineParser<>(TestRecordMissing.class, LOG_FORMAT);
        parser.parse(""); // Just to trigger the internal assembly of things (that should fail).
        fail("Missing exception.");
    } catch (MissingDissectorsException e) {
        assertTrue(e.getMessage().contains("HEADER:response.header.etag.thisshouldbemissing"));
    }
}
 

import nl.basjes.parse.core.Parser; //导入依赖的package包/类
@Test
public void testMissing2() throws Exception {
    try {
        Parser<TestRecordMissing2> parser = new HttpdLoglineParser<>(TestRecordMissing2.class, LOG_FORMAT);
        parser.parse(""); // Just to trigger the internal assembly of things (that should fail).
        fail("Missing exception.");
    } catch (MissingDissectorsException e) {
        assertTrue(e.getMessage().contains("BLURP:request.firstline.uri.query.thisshouldbemissing"));
    }
}