Python BaseThreadedModule.BaseThreadedModule类代码示例

 def initAfterFork(self):
     # As the buffer uses a threaded timed function to flush its buffer and thread will not survive a fork, init buffer here.
     self.buffers = collections.defaultdict(lambda: Buffer(flush_size=self.buffer_size,
                                                           callback=self.sendMergedEvent,
                                                           interval=self.flush_interval_in_secs,
                                                           maxsize=self.buffer_size))
     BaseThreadedModule.initAfterFork(self)

 def configure(self, configuration):
     # Call parent configure method
     BaseThreadedModule.configure(self, configuration)
     # Set boto log level.
     logging.getLogger('boto3').setLevel(logging.CRITICAL)
     logging.getLogger('botocore').setLevel(logging.CRITICAL)
     self.sqs_queue_name = self.getConfigurationValue('queue')
     self.attribute_names = self.getConfigurationValue('attribute_names')
     self.message_attribute_names = self.getConfigurationValue('message_attribute_names')
     self.poll_interval = self.getConfigurationValue('poll_interval_in_secs')
     self.batch_size = self.getConfigurationValue('batch_size')
     try:
         self.sqs_client = boto3.client('sqs', region_name=self.getConfigurationValue('region'),
                                               api_version=None,
                                               use_ssl=True,
                                               verify=None,
                                               endpoint_url=None,
                                               aws_access_key_id=self.getConfigurationValue('aws_access_key_id'),
                                               aws_secret_access_key=self.getConfigurationValue('aws_secret_access_key'),
                                               aws_session_token=None,
                                               config=None)
     except:
         etype, evalue, etb = sys.exc_info()
         self.logger.error("Could not connect to sqs service. Exception: %s, Error: %s." % (etype, evalue))
         self.lumbermill.shutDown()
     try:
         self.sqs_queue_url = self.sqs_client.get_queue_url(QueueName=self.sqs_queue_name)['QueueUrl']
     except:
         etype, evalue, etb = sys.exc_info()
         self.logger.error("Could not get queue url for sqs queue %s. Exception: %s, Error: %s." % (self.sqs_queue_name, etype, evalue))
         self.lumbermill.shutDown()

 def configure(self, configuration):
     # Call parent configure method.
     BaseThreadedModule.configure(self, configuration)
     self.format = self.getConfigurationValue('format')
     self.collection = self.getConfigurationValue('collection')
     self.database = self.getConfigurationValue('database')
     self.doc_id_pattern = self.getConfigurationValue("doc_id")

 def configure(self, configuration):
     # Call parent configure method
     BaseThreadedModule.configure(self, configuration)
     self.pretty_print = self.getConfigurationValue('pretty_print')
     self.fields = self.getConfigurationValue('fields')
     self.format = self.getConfigurationValue('format')
     self.printing = False

 def configure(self, configuration):
      # Call parent configure method
     BaseThreadedModule.configure(self, configuration)
     if len(self.getConfigurationValue('cluster')) == 0:
         redis_store = self.getConfigurationValue('server')
         self.client = self.getRedisClient()
     else:
         redis_store = self.getConfigurationValue('cluster')
         self.client = self.getClusterRedisClient()
     try:
         self.client.ping()
     except:
         etype, evalue, etb = sys.exc_info()
         self.logger.error("Could not connect to redis store at %s. Exception: %s, Error: %s." % (redis_store,etype, evalue))
         self.lumbermill.shutDown()
     self.set_buffer = None
     if self.getConfigurationValue('store_interval_in_secs') or self.getConfigurationValue('batch_size'):
         self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self.setBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
         self._set = self.set
         self.set = self.setBuffered
         self._get = self.get
         self.get = self.getBuffered
         self._delete = self.delete
         self.delete = self.deleteBuffered
         self._pop = self.pop
         self.pop = self.popBuffered

 def configure(self, configuration):
     # self.logger.setLevel(logging.DEBUG)
     # Call parent configure method.
     BaseThreadedModule.configure(self, configuration)
     self.is_leader = True if self.getConfigurationValue('pack') == 'leader' else False
     self.pack_followers = {}
     self.cluster_name = self.getConfigurationValue('name')
     self.discovered_leader = None
     self.secret = hashlib.sha256(self.getConfigurationValue('secret')).digest()
     self.handlers = collections.defaultdict(list)
     self.lock = threading.Lock()
     # Setup socket.
     self.interface_addr = self.getConfigurationValue('interface')
     self.interface_port = self.getConfigurationValue('port')
     self.broadcast_addr = self.getConfigurationValue('broadcast')
     self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
     self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
     self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
     self.socket.settimeout(1)
     self.hostname = socket.gethostname()
     try:
         self.socket.bind((self.interface_addr, self.interface_port))
     except:
         etype, evalue, etb = sys.exc_info()
         self.logger.error("Could not listen on %s:%s. Exception: %s, Error: %s." % (self.getConfigurationValue("interface"),
                                                                                     self.getConfigurationValue("port"), etype, evalue))
         self.alive = False
         self.lumbermill.shutDown()
         return
     self.addHandlers()

 def configure(self, configuration):
     # Call parent configure method.
     BaseThreadedModule.configure(self, configuration)
     for module_name in ['elasticsearch', 'urllib3', 'requests']:
         if self.getConfigurationValue('log_level') == 'info':
             logging.getLogger(module_name).setLevel(logging.WARN)
         else:
             # Set log level for elasticsarch library if configured to other than default.
             logging.getLogger(module_name).setLevel(self.logger.level)
     self.action = self.getConfigurationValue('action')
     self.fields = self.getConfigurationValue('fields')
     self.ttl = self.getConfigurationValue("ttl")
     self.index_name = self.getConfigurationValue("index_name")
     self.routing_pattern = self.getConfigurationValue("routing")
     self.doc_id_pattern = self.getConfigurationValue("doc_id")
     self.doc_type_pattern = self.getConfigurationValue("doc_type")
     self.doc_type_is_dynamic = self.isDynamicConfigurationValue("doc_type")
     self.es_nodes = self.getConfigurationValue("nodes")
     self.read_timeout = self.getConfigurationValue("read_timeout")
     if not isinstance(self.es_nodes, list):
         self.es_nodes = [self.es_nodes]
     if self.getConfigurationValue("connection_type") == 'urllib3':
         self.connection_class = elasticsearch.connection.Urllib3HttpConnection
     elif self.getConfigurationValue("connection_type") == 'requests':
         self.connection_class = elasticsearch.connection.RequestsHttpConnection

 def shutDown(self):
     if self.server:
         self.server.stop()
         # Give os time to free the socket. Otherwise a reload will fail with 'address already in use'
         time.sleep(.2)
     # Call parent shutDown method.
     BaseThreadedModule.shutDown(self)

 def initAfterFork(self):
     self.evaluate_facet_data_func = setInterval(self.getConfigurationValue('interval'))(self.evaluateFacets) #self.getEvaluateFunc()
     self.timed_func_handler_a = TimedFunctionManager.startTimedFunction(self.evaluate_facet_data_func)
     if self.cache:
         self.store_facets_in_cache_func = setInterval(1)(self.storeFacetsInCache)
         self.timed_func_handler_b = TimedFunctionManager.startTimedFunction(self.store_facets_in_cache_func)
     BaseThreadedModule.initAfterFork(self)

    def configure(self, configuration):
        BaseThreadedModule.configure(self, configuration)
        self.hostname = self.getConfigurationValue("hostname")
        self.fields = self.getConfigurationValue("fields")
        self.field_prefix = self.getConfigurationValue("field_prefix")
        self.timestamp_field = self.getConfigurationValue("timestamp_field")
        self.batch_size = self.getConfigurationValue('batch_size')
        self.backlog_size = self.getConfigurationValue('backlog_size')
        self.agent_conf = self.getConfigurationValue("agent_conf")
        if self.agent_conf:
            if self.agent_conf is True:
                self.agent_conf = "/etc/zabbix/zabbix_agentd.conf"
            if not os.path.isfile(self.agent_conf):
                self.logger.error("%s does not point to an existing file." % self.agent_conf)
                self.lumbermill.shutDown()
            self.zabbix_sender = ZabbixSender(use_config=self.agent_conf)

        else:
            self.logger.error("asdads")
            server = self.getConfigurationValue("server")
            port = 10051
            if ":" in self.server:
                server, port = self.server.split(":")
            self.zabbix_sender = ZabbixSender(zabbix_server=server, port=port)
        self.buffer = Buffer(self.getConfigurationValue('batch_size'), self.storeData,
                             self.getConfigurationValue('store_interval_in_secs'),
                             maxsize=self.getConfigurationValue('backlog_size'))

 def configure(self, configuration):
     # Call parent configure method
     BaseThreadedModule.configure(self, configuration)
     self.interval = self.getConfigurationValue('interval')
     self.fields = self.getConfigurationValue('fields')
     self.stats_collector = StatisticCollector()
     self.module_queues = {}

 def shutDown(self):
     self.accumulateReceiveRateStats()
     self.accumulateEventTypeStats()
     if self.lumbermill.is_master():
         self.printIntervalStatistics()
     self.mp_stats_collector.shutDown()
     BaseThreadedModule.shutDown(self)

    def configure(self, configuration):
        # Call parent configure method
        BaseThreadedModule.configure(self, configuration)
        self.backend = self.getConfigurationValue('backend')
        self.backend_client = None
        self.kv_store = None
        self.set_buffer = None
        if self.backend == 'DictStore':
            import simplekv.memory
            self.kv_store = simplekv.memory.DictStore()
        elif self.backend == 'RedisStore':
            import simplekv.memory.redisstore
            self.backend_client = self._getRedisClient()
            self.kv_store = simplekv.memory.redisstore.RedisStore(self.backend_client)
        elif self.backend == 'MemcacheStore':
            import simplekv.memory.memcachestore
            self.backend_client = self._getMemcacheClient()
            self.kv_store = simplekv.memory.memcachestore.MemcacheStore(self.backend_client)
        else:
            self.logger("Unknown backend type %s. Please check." % backend)
            self.lumbermill.shutDown();

        if self.getConfigurationValue('store_interval_in_secs') or self.getConfigurationValue('batch_size'):
            if self.backend == 'RedisStore':
                self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self._setRedisBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
            else:
                self.set_buffer = Buffer(self.getConfigurationValue('batch_size'), self._setBufferedCallback, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
            self._set = self.set
            self.set = self._setBuffered
            self._get = self.get
            self.get = self._getBuffered
            self._delete = self.delete
            self.delete = self._deleteBuffered
            self._pop = self.pop
            self.pop = self._popBuffered

 def configure(self, configuration):
      # Call parent configure method
     BaseThreadedModule.configure(self, configuration)
     # Make urllib3s logging less verbose.
     urllib3_logger = logging.getLogger('requests.packages.urllib3.connectionpool')
     urllib3_logger.setLevel(logging.CRITICAL)
     self.server, self.port = self.getConfigurationValue('server').split(':')
     self.user = self.getConfigurationValue('user')
     self.events_container = []
     self.batch_size = self.getConfigurationValue('batch_size')
     self.backlog_size = self.getConfigurationValue('backlog_size')
     self.path = self.getConfigurationValue('path')
     self.name_pattern = self.getConfigurationValue('name_pattern')
     self.format = self.getConfigurationValue('format')
     self.compress = self.getConfigurationValue('compress')
     if self.compress == 'gzip':
         try:
             import gzip
         except ImportError:
             self.logger.error('Gzip compression selected but gzip module could not be loaded.')
             self.lumbermill.shutDown()
     if self.compress == 'snappy':
         try:
             import snappy
         except ImportError:
             self.logger.error('Snappy compression selected but snappy module could not be loaded.')
             self.lumbermill.shutDown()
     self.is_storing = False
     self.lock = multiprocessing.Lock()
     self.timed_store_func = self.getTimedStoreFunc()

 def configure(self, configuration):
     # Call parent configure method
     BaseThreadedModule.configure(self, configuration)
     self.source_field = self.getConfigurationValue('source_field')
     self.seperator = self.getConfigurationValue('seperator')
     self.target_field = self.getConfigurationValue('target_field')
     self.drop_original = not self.getConfigurationValue('keep_original')