Python utils.DictUtils类代码示例

 def testAnonymizeMd5(self):
     self.test_object.configure({'action': 'anonymize',
                                 'source_fields': ['anon_me'],
                                 'algorithm': 'md5'})
     expected = DictUtils.getDefaultEventDict({'lumbermill': {'event_id': 1}, 'anon_me': 'bb649c83dd1ea5c9d9dec9a18df0ffe9'})
     for event in self.test_object.handleEvent(DictUtils.getDefaultEventDict({'lumbermill': {'event_id': 1}, 'anon_me': 'Nobody inspects the spammish repetition'})):
         self.assertEqual(event, expected)

 def processStatistics(self):
     stats_event = {"stats_type": "process_stats", "timestamp": time.time()}
     stats_event["worker_count"] = len(self.lumbermill.child_processes) + 1
     stats_event["uptime"] = int(time.time() - self.psutil_processes[0].create_time())
     self.logger.info(">> Process stats")
     self.logger.info("num workers: %d" % (len(self.lumbermill.child_processes)+1))
     self.logger.info("started: %s" % datetime.datetime.fromtimestamp(self.psutil_processes[0].create_time()).strftime("%Y-%m-%d %H:%M:%S"))
     aggregated_metrics = defaultdict(int)
     for psutil_process in self.psutil_processes:
         stats_event["pid"] = psutil_process.pid
         for metric_name, metric_value in psutil_process.as_dict(self.process_statistics).iteritems():
             # Call metric specific method if it exists.
             if "convertMetric_%s" % metric_name in self.methods:
                 metric_name, metric_value = getattr(self, "convertMetric_%s" % self.action)(metric_name, metric_value)
             try:
                 aggregated_metrics[metric_name] += metric_value
             except TypeError:
                 try:
                     metric_value = dict(metric_value.__dict__)
                 except:
                     pass
                 try:
                     stats_event[metric_name].append(metric_value)
                 except KeyError:
                     stats_event[metric_name] = [metric_value]
                 self.logger.info("%s(pid: %s): %s" % (metric_name, psutil_process.pid, metric_value))
         if self.emit_as_event:
             self.sendEvent(DictUtils.getDefaultEventDict(stats_event, caller_class_name="Statistics", event_type="statistic"))
     for agg_metric_name, agg_metric_value in aggregated_metrics.iteritems():
         self.logger.info("%s: %s" % (agg_metric_name, agg_metric_value))
     if self.emit_as_event:
         self.sendEvent(DictUtils.getDefaultEventDict(aggregated_metrics, caller_class_name="Statistics", event_type="statistic"))

 def testMd5Hash(self):
     self.test_object.configure({'action': 'hash',
                                 'source_fields': ['hash_me'],
                                 'target_fields': ['hash_me_hashed']})
     expected = DictUtils.getDefaultEventDict({'lumbermill': {'event_id': 1}, 'hash_me': 'Nobody inspects the spammish repetition', 'hash_me_hashed': 'bb649c83dd1ea5c9d9dec9a18df0ffe9'})
     for event in self.test_object.handleEvent(DictUtils.getDefaultEventDict({'lumbermill': {'event_id': 1}, 'hash_me': 'Nobody inspects the spammish repetition'})):
         self.assertEqual(event, expected)

 def testSha1Hash(self):
     self.test_object.configure({'action': 'hash',
                                 'algorithm': 'sha1',
                                 'source_fields': ['hash_me'],
                                 'target_fields': ['hash_me_hashed']})
     expected = DictUtils.getDefaultEventDict({'lumbermill': {'id': 1}, 'hash_me': 'Nobody inspects the spammish repetition', 'hash_me_hashed': '531b07a0f5b66477a21742d2827176264f4bbfe2'})
     for event in self.test_object.handleEvent(DictUtils.getDefaultEventDict({'lumbermill': {'id': 1}, 'hash_me': 'Nobody inspects the spammish repetition'})):
         self.assertEqual(event, expected)

 def testNewlineEndEvent(self):
     self.test_object.configure({'pattern': "\n$",
                                 'pattern_marks': 'EndOfEvent'})
     self.checkConfiguration()
     self.test_object.initAfterFork()
     event = DictUtils.getDefaultEventDict({'data': 'No newline.'}, received_from='TestMergeEvent_%s' % os.getpid())
     self.test_object.receiveEvent(event)
     event = DictUtils.getDefaultEventDict({'data': "But now: \n"}, received_from='TestMergeEvent_%s' % os.getpid())
     self.test_object.receiveEvent(event)
     time.sleep(1.5)
     events = []
     for event in self.receiver.getEvent():
         events.append(event)
     self.assertEquals(len(events), 1)
     self.assertEquals(events[0]['data'], 'No newline.But now: \n')

 def testSQSSink(self):
     self.test_object.configure({'aws_access_key_id': os.environ['AWS_ID'],
                                 'aws_secret_access_key': os.environ['AWS_KEY'],
                                 'region': 'eu-west-1',
                                 'queue': self.queue_name})
     self.checkConfiguration()
     self.test_object.initAfterFork()
     # Send some messages to the test queue.
     for _ in range(0, 100):
         event = DictUtils.getDefaultEventDict({u'data': u"You get 'Gone with the Wind', 'Les Miserables' by Victor Hugo, "
                                                     u"'The French Lieutenant's Woman' and with every third book you get dung."})
         self.test_object.receiveEvent(event)
     self.test_object.shutDown()
     # Give messages some time to arrive.
     time.sleep(2)
     # Get messages from queue
     messages = []
     for _ in range(0, 50):
         response = self.sqs_client.receive_message(QueueUrl=self.sqs_queue.url,
                                                    MaxNumberOfMessages=10)
         if not 'Messages' in response:
             break
         for message in response['Messages']:
             messages.append(message)
     self.assertEqual(len(messages), 100)
     self.assertEqual(json.loads(messages[0]['Body'])['data'], event['data'])

 def testUserAgentTargetField(self):
     self.test_object.configure({'source_fields': 'user_agent',
                                 'target_field': 'http_user_agent_data'})
     self.checkConfiguration()
     event = DictUtils.getDefaultEventDict({'user_agent': "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"})
     for event in self.test_object.handleEvent(event):
         self.assert_('http_user_agent_data' in event and event['http_user_agent_data']['device']['family'] == "Spider")

 def run(self):
     while self.alive:
         packet = None
         try:
             pcap_header, packet = self.sniffer.next()
         except:
             pass
         if not packet:
             continue
         decoder = self.getPacketDecoder('eth')
         if not decoder:
             continue
         decoded_data = {'protocols': []}
         for decoded_packet in decoder.decodePacket(packet):
             packet_type = str(type(decoded_packet))
             if packet_type == "<class 'impacket.ImpactPacket.Ethernet'>":
                 self.parseEtherPacket(decoded_packet, decoded_data)
             elif packet_type == "<class 'impacket.ImpactPacket.IP'>":
                 self.parseIPPacketEvent(decoded_packet, decoded_data)
             elif packet_type == "<class 'impacket.ImpactPacket.TCP'>":
                 self.parseTCPPacketEvent(decoded_packet, decoded_data)
             elif packet_type == "<class 'impacket.ImpactPacket.Data'>":
                 self.parseDataPacketEvent(decoded_packet, decoded_data)
         if decoded_data['data']:
             event = DictUtils.getDefaultEventDict(caller_class_name=self.__class__.__name__)
             if self.target_field:
                 event[self.target_field] = decoded_data
             else:
                 event.update(decoded_data)
             self.sendEvent(event)

 def run(self):
     while self.alive:
         for kafka_event in self.consumer:
             event = DictUtils.getDefaultEventDict(dict={"topic": kafka_event.topic, "data": kafka_event.value}, caller_class_name=self.__class__.__name__)
             self.sendEvent(event)
             if(self.auto_commit_enable):
                 self.consumer.task_done(kafka_event)

 def handleFileChange(self, callback_data):
     while True:
         try:
             line = callback_data['lines'].popleft()
         except IndexError:
             break
         self.sendEvent(DictUtils.getDefaultEventDict(dict={"filename": callback_data['filename'], "data": line}, caller_class_name=self.__class__.__name__))

 def testAddGeoInfoFromListField(self):
     self.test_object.configure({'geo_info_fields': ['country_code'],
                                 'source_fields': ['x_forwarded_for']})
     self.checkConfiguration()
     dict = DictUtils.getDefaultEventDict({'x_forwarded_for': ['99.124.167.129']})
     for event in self.test_object.handleEvent(dict):
         self.assertEqual(event['geo_info']['country_code'], 'US')

 def TestATcpConnection(self):
     print("testTcpConnection")
     self.test_object.configure({'port': 5353,
                                 'simple_separator': '\n'})
     self.checkConfiguration()
     self.test_object.initAfterFork()
     self.startTornadoEventLoop()
     # Give server process time to startup.
     time.sleep(.1)
     try:
         s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
         s.settimeout(1)
         s.connect(('localhost', self.test_object.getConfigurationValue('port')))
         for _ in range(0, 1500):
             s.sendall("Beethoven, Mozart, Chopin, Liszt, Brahms, Panties...I'm sorry...Schumann, Schubert, Mendelssohn and Bach. Names that will live for ever.\n")
         s.shutdown(socket.SHUT_RDWR)
         s.close()
         connection_succeeded = True
     except:
         etype, evalue, etb = sys.exc_info()
         print "Could not connect to %s:%s. Exception: %s, Error: %s" % ('localhost', self.test_object.getConfigurationValue("port"), etype, evalue)
         connection_succeeded = False
     self.assertTrue(connection_succeeded)
     expected_ret_val = DictUtils.getDefaultEventDict({'data': "Beethoven, Mozart, Chopin, Liszt, Brahms, Panties...I'm sorry...Schumann, Schubert, Mendelssohn and Bach. Names that will live for ever."})
     expected_ret_val.pop('lumbermill')
     event = False
     time.sleep(2)
     counter = 0
     for event in self.receiver.getEvent():
         counter += 1
     self.assertTrue(event is not False)
     self.assertEqual(counter, 1500)
     event.pop('lumbermill')
     self.assertDictEqual(event, expected_ret_val)
     self.tearDown()

 def testUnixSocket(self):
     self.test_object.configure({'path_to_socket': '/tmp/test.sock'})
     raise unittest.SkipTest('Skipping test because UnixSocket input is currently broken.')
     try:
         os.remove('/tmp/test.sock')
     except OSError:
         pass
     self.assertFalse(os.path.exists('/tmp/test.sock'))
     self.checkConfiguration()
     self.test_object.start()
     self.startTornadoEventLoop()
     time.sleep(.1)
     self.assertTrue(os.path.exists('/tmp/test.sock'))
     unix_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
     try:
         unix_socket.connect('/tmp/test.sock')
     except socket.errno:
         self.fail("Could not connect to unix socket.")
     for _ in range(0,5000):
         unix_socket.send(b"http://en.wikipedia.org/wiki/Monty_Python/?gambol=putty\r\n")
     expected_ret_val = DictUtils.getDefaultEventDict({'data': "http://en.wikipedia.org/wiki/Monty_Python/?gambol=putty\r\n"})
     expected_ret_val.pop('lumbermill')
     time.sleep(.5)
     event = False
     counter = 0
     for event in self.receiver.getEvent():
         counter += 1
     self.assertTrue(event)
     self.assertEqual(counter, 5000)
     event.pop('lumbermill')
     self.assertDictEqual(event, expected_ret_val)

 def testSelectedFields(self):
     self.test_object.configure({'nodes': [self.es_server],
                                 'fields': ['sheep'],
                                 'doc_id': '$(id)',
                                 'doc_type': '$(type)',
                                 'batch_size': 1})
     self.checkConfiguration()
     self.test_object.initAfterFork()
     timestring = datetime.datetime.utcnow().strftime('%Y.%m.%d')
     index_name = 'lumbermill-%s' % timestring
     try:
         self.es.indices.delete(index=index_name, ignore=[400, 404])
     except:
         pass
     self.es.indices.create(index=index_name)
     event = DictUtils.getDefaultEventDict({'McTeagle': "But it was with more simple, homespun verses that McTeagle's unique style first flowered.",
                                            'sheep': {'flying': 'scotsman',
                                                      'id': '12345',
                                                      'type': 'pirate'}})
     doc_id = event['sheep.id']
     self.test_object.receiveEvent(event)
     self.test_object.shutDown()
     time.sleep(1)
     try:
         result = self.es.get(index=index_name, doc_type='pirate', id=doc_id)
     except elasticsearch.exceptions.NotFoundError, e:
         self.fail(e)

 def __testStorageTTL(self):
     """
     Does not seem to be testable without waiting for at least 60 seconds.
     That seems to be the smallest interval the purger thread is running, no matter what I set ttl.interval to.
     The documentation @http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-indices.html#indices-ttl
     does not say anything about a lower limit but testing leads me to the assumption that 60s is the lowest limit.
     """
     self.test_object.configure({'nodes': [self.es_server],
                                 'index_name': self.test_index_name,
                                 'ttl': 100,
                                 'sniff_on_start': False,
                                 'store_interval_in_secs': 1})
     self.checkConfiguration()
     self.test_object.initAfterFork()
     # Enable ttl mapping.
     self.es.indices.close(index=self.test_index_name)
     self.es.indices.put_settings(index=self.test_index_name, body='{"ttl": {"interval" : "1s"}}')
     self.es.indices.open(index=self.test_index_name)
     self.es.indices.put_mapping(index=self.test_index_name, doc_type='Unknown', body='{"_ttl" : { "enabled" : true }}')
     event = DictUtils.getDefaultEventDict({'McTeagle': "But it was with more simple, homespun verses that McTeagle's unique style first flowered."})
     doc_id = event['lumbermill']['event_id']
     self.test_object.receiveEvent(event)
     self.test_object.shutDown()
     try:
         result = self.es.get(index=self.test_index_name, doc_type='Unknown', id=doc_id)
     except elasticsearch.NotFoundError:
         self.fail("Document was not found.")
     self.assertEqual(type(result), dict)
     self.assertDictContainsSubset(event, result['_source'])
     time.sleep(2)
     try:
         result = self.es.get(index=self.test_index_name, doc_type='Unknown', id=doc_id)
         self.fail("Document was not deleted after ttl.")
     except elasticsearch.NotFoundError:
         pass