Java Fields类代码示例

import org.kitesdk.morphline.base.Fields; //导入依赖的package包/类
@Test
public void testReadClob() throws Exception {
  Context context = new Context();
  context.put(MorphlineHandlerImpl.MORPHLINE_FILE_PARAM,
              RESOURCES_DIR + "/test-morphlines/readClob.conf");
  Event input = EventBuilder.withBody("foo", Charsets.UTF_8);
  input.getHeaders().put("name", "nadja");
  Event actual = build(context).intercept(input);
  Event expected = EventBuilder.withBody(null,
                                         ImmutableMap.of("name", "nadja", Fields.MESSAGE, "foo"));
  assertEqualsEvent(expected, actual);

  List<Event> actualList = build(context).intercept(Collections.singletonList(input));
  List<Event> expectedList = Collections.singletonList(expected);
  assertEqualsEventList(expectedList, actualList);
}
 

import org.kitesdk.morphline.base.Fields; //导入依赖的package包/类
@Test
public void testGrokIfNotMatchDropEventRetain() throws Exception {
  Context context = new Context();
  context.put(MorphlineHandlerImpl.MORPHLINE_FILE_PARAM,
              RESOURCES_DIR + "/test-morphlines/grokIfNotMatchDropRecord.conf");

  String msg = "<164>Feb  4 10:46:14 syslog sshd[607]: Server listening on 0.0.0.0 port 22.";
  Event input = EventBuilder.withBody(null, ImmutableMap.of(Fields.MESSAGE, msg));
  Event actual = build(context).intercept(input);

  Map<String, String> expected = new HashMap();
  expected.put(Fields.MESSAGE, msg);
  expected.put("syslog_pri", "164");
  expected.put("syslog_timestamp", "Feb  4 10:46:14");
  expected.put("syslog_hostname", "syslog");
  expected.put("syslog_program", "sshd");
  expected.put("syslog_pid", "607");
  expected.put("syslog_message", "Server listening on 0.0.0.0 port 22.");
  Event expectedEvent = EventBuilder.withBody(null, expected);
  assertEqualsEvent(expectedEvent, actual);
}
 

import org.kitesdk.morphline.base.Fields; //导入依赖的package包/类
@Test
/** morphline says route to southpole if it's an avro file, otherwise route to northpole */
public void testIfDetectMimeTypeRouteToSouthPole() throws Exception {
  Context context = new Context();
  context.put(MorphlineHandlerImpl.MORPHLINE_FILE_PARAM,
              RESOURCES_DIR + "/test-morphlines/ifDetectMimeType.conf");
  context.put(MorphlineHandlerImpl.MORPHLINE_VARIABLE_PARAM + ".MY.MIME_TYPE", "avro/binary");

  Event input = EventBuilder.withBody(Files.toByteArray(
      new File(RESOURCES_DIR + "/test-documents/sample-statuses-20120906-141433.avro")));
  Event actual = build(context).intercept(input);

  Map<String, String> expected = new HashMap();
  expected.put(Fields.ATTACHMENT_MIME_TYPE, "avro/binary");
  expected.put("flume.selector.header", "goToSouthPole");
  Event expectedEvent = EventBuilder.withBody(input.getBody(), expected);
  assertEqualsEvent(expectedEvent, actual);
}
 

import org.kitesdk.morphline.base.Fields; //导入依赖的package包/类
@Test
/** morphline says route to southpole if it's an avro file, otherwise route to northpole */
public void testIfDetectMimeTypeRouteToNorthPole() throws Exception {
  Context context = new Context();
  context.put(MorphlineHandlerImpl.MORPHLINE_FILE_PARAM,
              RESOURCES_DIR + "/test-morphlines/ifDetectMimeType.conf");
  context.put(MorphlineHandlerImpl.MORPHLINE_VARIABLE_PARAM + ".MY.MIME_TYPE", "avro/binary");

  Event input = EventBuilder.withBody(
      Files.toByteArray(new File(RESOURCES_DIR + "/test-documents/testPDF.pdf")));
  Event actual = build(context).intercept(input);

  Map<String, String> expected = new HashMap();
  expected.put(Fields.ATTACHMENT_MIME_TYPE, "application/pdf");
  expected.put("flume.selector.header", "goToNorthPole");
  Event expectedEvent = EventBuilder.withBody(input.getBody(), expected);
  assertEqualsEvent(expectedEvent, actual);
}
 

import org.kitesdk.morphline.base.Fields; //导入依赖的package包/类
public GenerateSolrSequenceKey(CommandBuilder builder, Config config, Command parent, Command child, MorphlineContext context) {
  super(builder, config, parent, child, context);
  this.baseIdFieldName = getConfigs().getString(config, "baseIdField", Fields.BASE_ID);
  this.preserveExisting = getConfigs().getBoolean(config, "preserveExisting", true);      
  
  Config solrLocatorConfig = getConfigs().getConfig(config, "solrLocator");
  SolrLocator locator = new SolrLocator(solrLocatorConfig, context);
  LOG.debug("solrLocator: {}", locator);
  IndexSchema schema = locator.getIndexSchema();
  SchemaField uniqueKey = schema.getUniqueKeyField();
  uniqueKeyName = uniqueKey == null ? null : uniqueKey.getName();
  
  String tmpIdPrefix = getConfigs().getString(config, "idPrefix", null);  // for load testing only
  Random tmpRandomIdPrefx = null;
  if ("random".equals(tmpIdPrefix)) { // for load testing only
    tmpRandomIdPrefx = new Random(new SecureRandom().nextLong());    
    tmpIdPrefix = null;
  }
  idPrefix = tmpIdPrefix;
  randomIdPrefix = tmpRandomIdPrefx;
  validateArguments();
}
 

import org.kitesdk.morphline.base.Fields; //导入依赖的package包/类
@Test
public void testLoadSolrBasic() throws Exception {
  //System.setProperty("ENV_SOLR_HOME", testSolrHome + "/collection1");
  morphline = createMorphline("test-morphlines/loadSolrBasic");    
  //System.clearProperty("ENV_SOLR_HOME");
  Record record = new Record();
  record.put(Fields.ID, "id0");
  record.put("first_name", "Nadja"); // will be sanitized
  startSession();
  Notifications.notifyBeginTransaction(morphline);
  assertTrue(morphline.process(record));
  assertEquals(1, collector.getNumStartEvents());
  Notifications.notifyCommitTransaction(morphline);
  Record expected = new Record();
  expected.put(Fields.ID, "id0");
  assertEquals(Arrays.asList(expected), collector.getRecords());
  assertEquals(1, queryResultSetSize("*:*"));
  Notifications.notifyRollbackTransaction(morphline);
  Notifications.notifyShutdown(morphline);
}
 

import org.kitesdk.morphline.base.Fields; //导入依赖的package包/类
@Test
public void testTokenizeText() throws Exception {
  morphline = createMorphline("test-morphlines" + File.separator + "tokenizeText");
  for (int i = 0; i < 3; i++) {
    Record record = new Record();
    record.put(Fields.MESSAGE, "Hello World!");
    record.put(Fields.MESSAGE, "\[email protected] #%()123");
    Record expected = record.copy();
    expected.getFields().putAll("tokens", Arrays.asList("hello", "world", "foo", "bar.com", "123"));
    collector.reset();
    startSession();
    Notifications.notifyBeginTransaction(morphline);
    assertTrue(morphline.process(record));
    assertEquals(1, collector.getNumStartEvents());
    Notifications.notifyCommitTransaction(morphline);
    assertEquals(expected, collector.getFirstRecord());
  }
}
 

import org.kitesdk.morphline.base.Fields; //导入依赖的package包/类
@Test
public void testGrokSyslogNgCisco() throws Exception {
  morphline = createMorphline("test-morphlines/grokSyslogNgCisco");
  Record record = new Record();
  String msg = "<179>Jun 10 04:42:51 www.foo.com Jun 10 2013 04:42:51 : %myproduct-3-mysubfacility-251010: " +
      "Health probe failed for server 1.2.3.4 on port 8083, connection refused by server";
  record.put(Fields.MESSAGE, msg);
  assertTrue(morphline.process(record));
  Record expected = new Record();
  expected.put(Fields.MESSAGE, msg);
  expected.put("cisco_message_code", "%myproduct-3-mysubfacility-251010");
  expected.put("cisco_product", "myproduct");
  expected.put("cisco_level", "3");
  expected.put("cisco_subfacility", "mysubfacility");
  expected.put("cisco_message_id", "251010");
  expected.put("syslog_message", "%myproduct-3-mysubfacility-251010: Health probe failed for server 1.2.3.4 " +
      "on port 8083, connection refused by server");
  assertEquals(expected, collector.getFirstRecord());
  assertNotSame(record, collector.getFirstRecord());      
}
 

import org.kitesdk.morphline.base.Fields; //导入依赖的package包/类
public void testGrokSyslogNgCiscoWithoutSubFacility() throws Exception {
    morphline = createMorphline("test-morphlines/grokSyslogNgCisco");
    Record record = new Record();
    String msg = "<179>Jun 10 04:42:51 www.foo.com Jun 10 2013 04:42:51 : %myproduct-3-mysubfacility-251010: " +
        "Health probe failed for server 1.2.3.4 on port 8083, connection refused by server";
    record.put(Fields.MESSAGE, msg);
    assertTrue(morphline.process(record));
    Record expected = new Record();
    expected.put(Fields.MESSAGE, msg);
    expected.put("cisco_message_code", "%myproduct-3-251010");
    expected.put("cisco_product", "myproduct");
    expected.put("cisco_level", "3");
//    expected.put("cisco_subfacility", "mysubfacility");
    expected.put("cisco_message_id", "251010");
    expected.put("syslog_message", "%myproduct-3-mysubfacility-251010: Health probe failed for server 1.2.3.4 " +
        "on port 8083, connection refused by server");
    assertEquals(expected, collector.getFirstRecord());
    assertNotSame(record, collector.getFirstRecord());      
  }
 

import org.kitesdk.morphline.base.Fields; //导入依赖的package包/类
@Test
public void testGrokEmail() throws Exception {
  morphline = createMorphline("test-morphlines/grokEmail");
  Record record = new Record();
  byte[] bytes = Files.toByteArray(new File(RESOURCES_DIR + "/test-documents/email.txt"));
  record.put(Fields.ATTACHMENT_BODY, bytes);
  assertTrue(morphline.process(record));
  Record expected = new Record();
  String msg = new String(bytes, "UTF-8"); //.replaceAll("(\r)?\n", "\n");
  expected.put(Fields.MESSAGE, msg);
  expected.put("message_id", "[email protected]");
  expected.put("date", "Wed, 6 Feb 2012 06:06:05 -0800");
  expected.put("from", "[email protected]");
  expected.put("to", "[email protected]");
  expected.put("subject", "WEDNESDAY WEATHER HEADLINES");
  expected.put("from_names", "Foo Bar <[email protected]>@xxx");
  expected.put("to_names", "'Weather News Distribution' <[email protected]>");    
  expected.put("text", 
      "Average 1 to 3- degrees above normal: Mid-Atlantic, Southern Plains.." +
      "\nAverage 4 to 6-degrees above normal: Ohio Valley, Rockies, Central Plains");
  assertEquals(expected, collector.getFirstRecord());
  assertNotSame(record, collector.getFirstRecord());      
}
 

import org.kitesdk.morphline.base.Fields; //导入依赖的package包/类
@Test
public void testExtractJsonPathsFlattened() throws Exception {
  morphline = createMorphline("test-morphlines/extractJsonPathsFlattened");    
  File file = new File(RESOURCES_DIR + "/test-documents/arrays.json");
  InputStream in = new BufferedInputStream(new FileInputStream(file));
  Record record = new Record();
  record.put(Fields.ATTACHMENT_BODY, in);
  
  startSession();
  assertEquals(1, collector.getNumStartEvents());
  assertTrue(morphline.process(record));    
  
  assertEquals(1, collector.getRecords().size());
  List expected = Arrays.asList(1, 2, 3, 4, 5, 10, 20, 100, 200);
  assertEquals(1, collector.getRecords().size());
  assertEquals(expected, collector.getFirstRecord().get("/price"));
  assertEquals(expected, collector.getFirstRecord().get("/price/[]"));
  assertEquals(Arrays.asList(), collector.getFirstRecord().get("/unknownField"));

  in.close();
}
 

import org.kitesdk.morphline.base.Fields; //导入依赖的package包/类
@Test
@Ignore
/** Crude quick n' dirty benchmark */
// Before running this disable debug logging 
// via log4j.logger.org.kitesdk.morphline=INFO in log4j.properties
public void benchmarkJson() throws Exception {
  String morphlineConfigFile = "test-morphlines/readJson";
  long durationSecs = 20;
  //File file = new File(RESOURCES_DIR + "/test-documents/stream.json");
  File file = new File(RESOURCES_DIR + "/test-documents/sample-statuses-20120906-141433.json");
  System.out.println("Now benchmarking " + morphlineConfigFile + " ...");
  morphline = createMorphline(morphlineConfigFile);    
  byte[] bytes = Files.toByteArray(file);
  long start = System.currentTimeMillis();
  long duration = durationSecs * 1000;
  int iters = 0; 
  while (System.currentTimeMillis() < start + duration) {
    Record record = new Record();
    record.put(Fields.ATTACHMENT_BODY, bytes);      
    collector.reset();
    assertTrue(morphline.process(record));    
    iters++;
  }
  float secs = (System.currentTimeMillis() - start) / 1000.0f;
  System.out.println("Results: iters=" + iters + ", took[secs]=" + secs + ", iters/secs=" + (iters/secs));
}
 

import org.kitesdk.morphline.base.Fields; //导入依赖的package包/类
@Override
public void process(Event event) {
  numRecords.mark();
  Timer.Context timerContext = mappingTimer.time();
  try {
    Record record = new Record();
    for (Entry<String, String> entry : event.getHeaders().entrySet()) {
      record.put(entry.getKey(), entry.getValue());
    }
    byte[] bytes = event.getBody();
    if (bytes != null && bytes.length > 0) {
      record.put(Fields.ATTACHMENT_BODY, bytes);
    }    
    try {
      Notifications.notifyStartSession(morphline);
      if (!morphline.process(record)) {
        numFailedRecords.mark();
        LOG.warn("Morphline {} failed to process record: {}", morphlineFileAndId, record);
      }
    } catch (RuntimeException t) {
      numExceptionRecords.mark();
      morphlineContext.getExceptionHandler().handleException(t, record);
    }
  } finally {
    timerContext.stop();
  }
}
 

import org.kitesdk.morphline.base.Fields; //导入依赖的package包/类
@Test
/* leading XXXXX does not match regex, thus we expect the event to be dropped */
public void testGrokIfNotMatchDropEventDrop() throws Exception {
  Context context = new Context();
  context.put(MorphlineHandlerImpl.MORPHLINE_FILE_PARAM,
              RESOURCES_DIR + "/test-morphlines/grokIfNotMatchDropRecord.conf");
  String msg = "<XXXXXXXXXXXXX164>Feb  4 10:46:14 syslog sshd[607]: Server listening on 0.0.0.0" +
                   " port 22.";
  Event input = EventBuilder.withBody(null, ImmutableMap.of(Fields.MESSAGE, msg));
  Event actual = build(context).intercept(input);
  assertNull(actual);
}
 

import org.kitesdk.morphline.base.Fields; //导入依赖的package包/类
private void testDocumentTypesInternal(String... files) throws Exception {
    int numDocs = 0;
    long startTime = System.currentTimeMillis();
    
    assertEquals(numDocs, queryResultSetSize("*:*"));      
//  assertQ(req("*:*"), "//*[@numFound='0']");
    for (int i = 0; i < 1; i++) {      
      for (String file : files) {
        File f = new File(file);
        byte[] body = Files.toByteArray(f);
        Event event = EventBuilder.withBody(body);
        event.getHeaders().put(Fields.ATTACHMENT_NAME, f.getName());
        load(event);
        Integer count = expectedRecords.get(file);
        if (count != null) {
          numDocs += count;
        } else {
          numDocs++;
        }
        assertEquals(numDocs, queryResultSetSize("*:*"));
      }
      LOGGER.trace("iter: {}", i);
    }
    LOGGER.trace("all done with put at {}", System.currentTimeMillis() - startTime);
    assertEquals(numDocs, queryResultSetSize("*:*"));
    LOGGER.trace("sink: ", sink);
  }