本文整理汇总了Python中cola.job.loader.LimitionJobLoader类的典型用法代码示例。如果您正苦于以下问题：Python LimitionJobLoader类的具体用法？Python LimitionJobLoader怎么用？Python LimitionJobLoader使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了LimitionJobLoader类的6个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: finish
def finish(self):
all_pages = self.pages()
self.release_lock(self.ready_lock)
self.release_lock(self.finish_lock)
LimitionJobLoader.finish(self)
JobLoader.finish(self)
self.stop_logger_server()
try:
for handler in self.logger.handlers:
handler.close()
except:
pass
if self.client is not None:
rpc_client = '%s:%s' % (
self.client.split(':')[0],
main_conf.client.port
)
client_call(rpc_client, 'stop', ignore=True)
self.logger.info('All nodes finishes visiting pages size: %s' % all_pages)
self.stopped = True示例2: __init__
def __init__(self, job, data_dir, nodes, client=None,
context=None, copies=1, force=False):
ctx = context or job.context
master_port = ctx.job.master_port
local = '%s:%s' % (get_ip(), master_port)
JobLoader.__init__(self, job, data_dir, local,
context=ctx, copies=copies, force=force)
LimitionJobLoader.__init__(self, job, context=ctx)
# check
self.check()
self.nodes = nodes
self.not_registered = self.nodes[:]
self.not_finished = self.nodes[:]
# mq
self.mq_client = MessageQueueClient(self.nodes, copies=copies)
# lock
self.ready_lock = threading.Lock()
self.ready_lock.acquire()
self.finish_lock = threading.Lock()
self.finish_lock.acquire()
# logger
self.logger = get_logger(
name='cola_master_%s'%self.job.real_name,
filename=os.path.join(self.root, 'job.log'),
is_master=True)
self.client = client
self.client_handler = None
if self.client is not None:
self.client_handler = add_log_client(self.logger, self.client)
self.init_rpc_server()
self.init_rate_clear()
self.init_logger_server(self.logger)
# register rpc server
self.rpc_server.register_function(self.client_stop, 'client_stop')
self.rpc_server.register_function(self.ready, 'ready')
self.rpc_server.register_function(self.worker_finish, 'worker_finish')
self.rpc_server.register_function(self.complete, 'complete')
self.rpc_server.register_function(self.error, 'error')
self.rpc_server.register_function(self.get_nodes, 'get_nodes')
self.rpc_server.register_function(self.apply, 'apply')
self.rpc_server.register_function(self.require, 'require')
self.rpc_server.register_function(self.stop, 'stop')
self.rpc_server.register_function(self.add_node, 'add_node')
self.rpc_server.register_function(self.remove_node, 'remove_node')
# register signal
signal.signal(signal.SIGINT, self.signal_handler)
signal.signal(signal.SIGTERM, self.signal_handler)示例3: __init__
def __init__(self, job, data_dir, master=None, local=None, nodes=None,
context=None, logger=None, copies=1, force=False):
BasicWorkerJobLoader.__init__(self, job, data_dir, context=context, logger=logger,
local=local, nodes=nodes, copies=copies, force=force)
LimitionJobLoader.__init__(self, self.job, context=context)
if self.logger is None:
self.logger = get_logger(
name='cola_worker_%s'%self.job.real_name,
filename=os.path.join(self.root, 'job.log'))
self.init_rate_clear()示例4: error
def error(self, obj):
LimitionJobLoader.error(self, obj)
BasicWorkerJobLoader.error(self, obj)示例5: complete
def complete(self, obj):
BasicWorkerJobLoader.complete(self, obj)
return LimitionJobLoader.complete(self, obj)示例6: stop
def stop(self):
LimitionJobLoader.stop(self)
BasicWorkerJobLoader.stop(self)