Python BaseThreadedModule.initAfterFork方法代码示例

# 需要导入模块: from lumbermill.BaseThreadedModule import BaseThreadedModule [as 别名]
# 或者: from lumbermill.BaseThreadedModule.BaseThreadedModule import initAfterFork [as 别名]
 def initAfterFork(self):
     # As the buffer uses a threaded timed function to flush its buffer and thread will not survive a fork, init buffer here.
     self.buffers = collections.defaultdict(lambda: Buffer(flush_size=self.buffer_size,
                                                           callback=self.sendMergedEvent,
                                                           interval=self.flush_interval_in_secs,
                                                           maxsize=self.buffer_size))
     BaseThreadedModule.initAfterFork(self)

# 需要导入模块: from lumbermill.BaseThreadedModule import BaseThreadedModule [as 别名]
# 或者: from lumbermill.BaseThreadedModule.BaseThreadedModule import initAfterFork [as 别名]
 def initAfterFork(self):
     self.evaluate_facet_data_func = setInterval(self.getConfigurationValue('interval'))(self.evaluateFacets) #self.getEvaluateFunc()
     self.timed_func_handler_a = TimedFunctionManager.startTimedFunction(self.evaluate_facet_data_func)
     if self.cache:
         self.store_facets_in_cache_func = setInterval(1)(self.storeFacetsInCache)
         self.timed_func_handler_b = TimedFunctionManager.startTimedFunction(self.store_facets_in_cache_func)
     BaseThreadedModule.initAfterFork(self)

# 需要导入模块: from lumbermill.BaseThreadedModule import BaseThreadedModule [as 别名]
# 或者: from lumbermill.BaseThreadedModule.BaseThreadedModule import initAfterFork [as 别名]
 def initAfterFork(self):
     BaseThreadedModule.initAfterFork(self)
     self.buffer = Buffer(self.getConfigurationValue('batch_size'), self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))
     self.connection = self.connect()
     if not self.connection:
         self.lumbermill.shutDown()
         return
     BaseThreadedModule.initAfterFork(self)

# 需要导入模块: from lumbermill.BaseThreadedModule import BaseThreadedModule [as 别名]
# 或者: from lumbermill.BaseThreadedModule.BaseThreadedModule import initAfterFork [as 别名]
 def initAfterFork(self):
     # Get all configured queues for waiting event stats.
     self.module_queues = self.lumbermill.getAllQueues()
     self.psutil_processes.append(psutil.Process(self.lumbermill.getMainProcessId()))
     for worker in self.lumbermill.child_processes:
         self.psutil_processes.append(psutil.Process(worker.pid))
     TimedFunctionManager.startTimedFunction(self.getRunTimedFunctionsFunc())
     BaseThreadedModule.initAfterFork(self)

# 需要导入模块: from lumbermill.BaseThreadedModule import BaseThreadedModule [as 别名]
# 或者: from lumbermill.BaseThreadedModule.BaseThreadedModule import initAfterFork [as 别名]
 def initAfterFork(self):
     BaseThreadedModule.initAfterFork(self)
     # Init monogdb client after fork.
     self.mongodb = self.connect()
     if not self.mongodb:
         self.lumbermill.shutDown()
         return
     # As the buffer uses a threaded timed function to flush its buffer and thread will not survive a fork, init buffer here.
     self.buffer = Buffer(self.getConfigurationValue('batch_size'), self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))

# 需要导入模块: from lumbermill.BaseThreadedModule import BaseThreadedModule [as 别名]
# 或者: from lumbermill.BaseThreadedModule.BaseThreadedModule import initAfterFork [as 别名]
 def initAfterFork(self):
     # Get all configured queues for waiting event stats.
     for module_name, module_info in self.lumbermill.modules.items():
         instance = module_info['instances'][0]
         if not hasattr(instance, 'getInputQueue') or not instance.getInputQueue():
             continue
         self.module_queues[module_name] = instance.getInputQueue()
     TimedFunctionManager.startTimedFunction(self.getRunTimedFunctionsFunc())
     BaseThreadedModule.initAfterFork()

# 需要导入模块: from lumbermill.BaseThreadedModule import BaseThreadedModule [as 别名]
# 或者: from lumbermill.BaseThreadedModule.BaseThreadedModule import initAfterFork [as 别名]
 def initAfterFork(self):
     BaseThreadedModule.initAfterFork(self)
     # Init es client after fork as mentioned in https://elasticsearch-py.readthedocs.org/en/master/
     self.es = self.connect()
     if not self.es:
         self.lumbermill.shutDown()
         return
     # As the buffer uses a threaded timed function to flush its buffer and thread will not survive a fork, init buffer here.
     self.buffer = Buffer(self.getConfigurationValue('batch_size'), self.storeData, self.getConfigurationValue('store_interval_in_secs'), maxsize=self.getConfigurationValue('backlog_size'))

# 需要导入模块: from lumbermill.BaseThreadedModule import BaseThreadedModule [as 别名]
# 或者: from lumbermill.BaseThreadedModule.BaseThreadedModule import initAfterFork [as 别名]
 def initAfterFork(self):
     self.resolver = resolver.Resolver()
     self.resolver.timeout = self.timeout
     self.resolver.lifetime = self.timeout
     if self.nameservers:
         self.resolver.nameservers = self.nameservers
     self.queue = Queue.Queue(20)
     self.lookup_threads = [LookupThread(self.queue, self.lookup_type, self) for _ in range(0, self.lookup_threads_pool_size)]
     for thread in self.lookup_threads:
         thread.start()
     BaseThreadedModule.initAfterFork(self)

# 需要导入模块: from lumbermill.BaseThreadedModule import BaseThreadedModule [as 别名]
# 或者: from lumbermill.BaseThreadedModule.BaseThreadedModule import initAfterFork [as 别名]
 def initAfterFork(self):
     BaseThreadedModule.initAfterFork(self)
     # Calculate event count when running in multiple processes.
     if self.max_events_count == 0:
         return
     self.max_events_count = int(self.getConfigurationValue("events_count")/self.lumbermill.getWorkerCount())
     if self.lumbermill.is_master():
         remainder = self.getConfigurationValue("events_count") % self.lumbermill.getWorkerCount()
         self.max_events_count += remainder
     if self.max_events_count == 0:
         self.shutDown()

# 需要导入模块: from lumbermill.BaseThreadedModule import BaseThreadedModule [as 别名]
# 或者: from lumbermill.BaseThreadedModule.BaseThreadedModule import initAfterFork [as 别名]
 def initAfterFork(self):
     """
     When running with multiple processes, calculate the the number of files each processes should work on.
     If dividing the number of files by the number of workers yields a remainder, master process will take care of it.
     """
     BaseThreadedModule.initAfterFork(self)
     datastore = self.lumbermill.getInternalDataStore()
     datastore.acquireLock()
     from_file_list_idx = datastore.getDataDict()[self.datastore_key]
     to_file_list_idx = from_file_list_idx + int(len(self.files) / self.lumbermill.getWorkerCount())
     if self.lumbermill.is_master():
         to_file_list_idx += len(self.files) % self.lumbermill.getWorkerCount()
     self.files = self.files[from_file_list_idx:to_file_list_idx]
     datastore.getDataDict()[self.datastore_key] = to_file_list_idx
     datastore.releaseLock()

# 需要导入模块: from lumbermill.BaseThreadedModule import BaseThreadedModule [as 别名]
# 或者: from lumbermill.BaseThreadedModule.BaseThreadedModule import initAfterFork [as 别名]
 def initAfterFork(self):
     timed_func = self.getRunTimedFunctionsFunc()
     TimedFunctionManager.startTimedFunction(timed_func)
     BaseThreadedModule.initAfterFork(self)

# 需要导入模块: from lumbermill.BaseThreadedModule import BaseThreadedModule [as 别名]
# 或者: from lumbermill.BaseThreadedModule.BaseThreadedModule import initAfterFork [as 别名]
 def initAfterFork(self):
     if self.interval:
         self.evaluate_facet_data_func = self.getEvaluateFunc()
         self.timed_func_handler = TimedFunctionManager.startTimedFunction(self.evaluate_facet_data_func)
     BaseThreadedModule.initAfterFork(self)

# 需要导入模块: from lumbermill.BaseThreadedModule import BaseThreadedModule [as 别名]
# 或者: from lumbermill.BaseThreadedModule.BaseThreadedModule import initAfterFork [as 别名]
 def initAfterFork(self):
     self.hdfs = self.getHdfsClient()
     TimedFunctionManager.startTimedFunction(self.timed_store_func)
     # Call parent run method
     BaseThreadedModule.initAfterFork(self)

# 需要导入模块: from lumbermill.BaseThreadedModule import BaseThreadedModule [as 别名]
# 或者: from lumbermill.BaseThreadedModule.BaseThreadedModule import initAfterFork [as 别名]
 def initAfterFork(self):
     if self.hot_rules_first:
         resort_fieldextraction_regexpressions_func = self.getResortFieldextractionRegexpressionsFunc()
         self.timed_func_handler = TimedFunctionManager.startTimedFunction(resort_fieldextraction_regexpressions_func)
     BaseThreadedModule.initAfterFork(self)

# 需要导入模块: from lumbermill.BaseThreadedModule import BaseThreadedModule [as 别名]
# 或者: from lumbermill.BaseThreadedModule.BaseThreadedModule import initAfterFork [as 别名]
 def initAfterFork(self):
     if self.interval:
         TimedFunctionManager.startTimedFunction(self.getRunTimedFunctionsFunc())
     BaseThreadedModule.initAfterFork(self)