本文整理汇总了Python中zmq.eventloop.ioloop.IOLoop.start方法的典型用法代码示例。如果您正苦于以下问题:Python IOLoop.start方法的具体用法?Python IOLoop.start怎么用?Python IOLoop.start使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类zmq.eventloop.ioloop.IOLoop
的用法示例。
在下文中一共展示了IOLoop.start方法的7个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: AsyncServerAdapter
# 需要导入模块: from zmq.eventloop.ioloop import IOLoop [as 别名]
# 或者: from zmq.eventloop.ioloop.IOLoop import start [as 别名]
class AsyncServerAdapter(object):
producer_class = Producer
def __init__(self, backend_rep_uri, frontend_rep_uri, frontend_pub_uri,
control_pipe=None):
self.uris = OrderedDict([
('backend_rep', backend_rep_uri),
('consumer_push_be', unique_ipc_uri()),
('consumer_pull_be', unique_ipc_uri()),
('frontend_rep_uri', frontend_rep_uri),
('frontend_pub_uri', frontend_pub_uri)
])
self.control_pipe = control_pipe
self.done = False
logging.getLogger(log_label(self)).info("uris: %s", self.uris)
def watchdog(self):
if self.control_pipe is None:
return
elif not self.done and self.control_pipe.poll():
self.done = True
self.finish()
def run(self):
consumer = Process(target=Consumer(self.uris['backend_rep'],
self.uris['consumer_push_be'],
self.uris['consumer_pull_be']).run
)
producer = Process(target=self.producer_class(
self.uris['frontend_rep_uri'],
self.uris['frontend_pub_uri'],
self.uris['consumer_pull_be'],
self.uris['consumer_push_be']).run
)
self.io_loop = IOLoop()
periodic_callback = PeriodicCallback(self.watchdog, 500, self.io_loop)
periodic_callback.start()
try:
consumer.start()
producer.start()
self.io_loop.start()
except KeyboardInterrupt:
pass
producer.terminate()
consumer.terminate()
logging.getLogger(log_label(self)).info('PRODUCER and CONSUMER have '
'been terminated')
def __del__(self):
uris = [self.uris[label] for label in ('consumer_push_be',
'consumer_pull_be', )]
cleanup_ipc_uris(uris)
def finish(self):
logging.getLogger(log_label(self)).debug('"finish" request received')
self.io_loop.stop()
示例2: __init__
# 需要导入模块: from zmq.eventloop.ioloop import IOLoop [as 别名]
# 或者: from zmq.eventloop.ioloop.IOLoop import start [as 别名]
class Server:
_encoding = 'utf-8'
def __init__(self, application, settings):
self.host = settings.get('host', '*')
self.port = int(settings.get('port', 8195))
self.application = application
self._context = zmq.Context()
self._context.set(
zmq.MAX_SOCKETS,
int(settings.get('alpaca.connection_limit', 10000))
)
queued_message_limit = int(
settings.get('alpaca.queued_message_limit', 1000)
)
self._socket = self._context.socket(zmq.SUB)
self._socket.subscribe = bytes()
self._socket.rcvhwm = queued_message_limit * 2 # 2-part messages
self._io_loop = IOLoop()
self._zmq_stream = ZMQStream(self._socket, io_loop=self._io_loop)
self._zmq_stream.on_recv(self.application)
def serve(self):
self._socket.bind(
'tcp://{host}:{port}'.format(host=self.host, port=self.port)
)
logger.info("Listening on {host}:{port}...".format(
host=self.host,
port=self.port
))
self._io_loop.start()
示例3: IOPubThread
# 需要导入模块: from zmq.eventloop.ioloop import IOLoop [as 别名]
# 或者: from zmq.eventloop.ioloop.IOLoop import start [as 别名]
class IOPubThread(object):
"""An object for sending IOPub messages in a background thread
Prevents a blocking main thread from delaying output from threads.
IOPubThread(pub_socket).background_socket is a Socket-API-providing object
whose IO is always run in a thread.
"""
def __init__(self, socket, pipe=False):
"""Create IOPub thread
Parameters
----------
socket: zmq.PUB Socket
the socket on which messages will be sent.
pipe: bool
Whether this process should listen for IOPub messages
piped from subprocesses.
"""
self.socket = socket
self.background_socket = BackgroundSocket(self)
self._master_pid = os.getpid()
self._pipe_flag = pipe
self.io_loop = IOLoop(make_current=False)
if pipe:
self._setup_pipe_in()
self._local = threading.local()
self._events = deque()
self._setup_event_pipe()
self.thread = threading.Thread(target=self._thread_main)
self.thread.daemon = True
def _thread_main(self):
"""The inner loop that's actually run in a thread"""
self.io_loop.make_current()
self.io_loop.start()
self.io_loop.close(all_fds=True)
def _setup_event_pipe(self):
"""Create the PULL socket listening for events that should fire in this thread."""
ctx = self.socket.context
pipe_in = ctx.socket(zmq.PULL)
pipe_in.linger = 0
_uuid = b2a_hex(os.urandom(16)).decode('ascii')
iface = self._event_interface = 'inproc://%s' % _uuid
pipe_in.bind(iface)
self._event_puller = ZMQStream(pipe_in, self.io_loop)
self._event_puller.on_recv(self._handle_event)
@property
def _event_pipe(self):
"""thread-local event pipe for signaling events that should be processed in the thread"""
try:
event_pipe = self._local.event_pipe
except AttributeError:
# new thread, new event pipe
ctx = self.socket.context
event_pipe = ctx.socket(zmq.PUSH)
event_pipe.linger = 0
event_pipe.connect(self._event_interface)
self._local.event_pipe = event_pipe
return event_pipe
def _handle_event(self, msg):
"""Handle an event on the event pipe
Content of the message is ignored.
Whenever *an* event arrives on the event stream,
*all* waiting events are processed in order.
"""
# freeze event count so new writes don't extend the queue
# while we are processing
n_events = len(self._events)
for i in range(n_events):
event_f = self._events.popleft()
event_f()
def _setup_pipe_in(self):
"""setup listening pipe for IOPub from forked subprocesses"""
ctx = self.socket.context
# use UUID to authenticate pipe messages
self._pipe_uuid = os.urandom(16)
pipe_in = ctx.socket(zmq.PULL)
pipe_in.linger = 0
try:
self._pipe_port = pipe_in.bind_to_random_port("tcp://127.0.0.1")
except zmq.ZMQError as e:
warnings.warn("Couldn't bind IOPub Pipe to 127.0.0.1: %s" % e +
"\nsubprocess output will be unavailable."
)
self._pipe_flag = False
pipe_in.close()
return
#.........这里部分代码省略.........
示例4: WorkerConnection
# 需要导入模块: from zmq.eventloop.ioloop import IOLoop [as 别名]
# 或者: from zmq.eventloop.ioloop.IOLoop import start [as 别名]
class WorkerConnection(object):
def __init__(self, address, retry_ms=10000, compression=None, serialization=None):
self.address = address
self.message_address = 'inproc://WorkerConnection%s' % id(self)
self.__context = zmq.Context()
self.__queue_socket = self.__context.socket(zmq.PUSH)
self.__queue_socket.bind(self.message_address)
self.__thread = None
self.__retry_ms = retry_ms
self.__callbacks = {}
self.__queued_messages = {}
self.__message_timeouts = {}
self.__ioloop = None
self.loads = serialization and serialization.loads or pickle.loads
self.dumps = serialization and serialization.dumps or pickle.dumps
self.compress = compression and compression.compress or (lambda x: x)
self.decompress = compression and compression.decompress or (lambda x: x)
def invoke(self, method, parameters, callback=None, retry_ms=0):
self._queue_message(self.compress(self.dumps({'method': method, 'parameters': parameters})), callback, retry_ms)
def __len__(self):
return len(self.__queued_messages)
def __getattr__(self, path):
return WorkerInvocation(path, self)
def _queue_message(self, message, callback=None, retry_ms=0):
if not self.__ioloop:
self.start()
message_id = str(uuid4())
if callback:
self.__callbacks[message_id] = callback
if retry_ms > 0:
self.__message_timeouts[message_id] = retry_ms
self.__queue_socket.send_multipart(('', message_id, message))
def log_error(self, error):
logging.error(repr(error))
def start(self):
def loop():
self.__ioloop = IOLoop()
queue_socket = self.__context.socket(zmq.PULL)
queue_socket.connect(self.message_address)
queue_stream = ZMQStream(queue_socket, self.__ioloop)
worker_socket = self.__context.socket(zmq.DEALER)
worker_socket.connect(self.address)
worker_stream = ZMQStream(worker_socket, self.__ioloop)
def receive_response(message):
self.__queued_messages.pop(message[1], None)
self.__message_timeouts.pop(message[1], None)
callback = self.__callbacks.pop(message[1], None)
if callback:
try:
callback(self.loads(self.decompress(message[2])))
except Exception as e:
self.log_error(e)
worker_stream.on_recv(receive_response)
def queue_message(message):
self.__queued_messages[message[1]] = (time() * 1000, message)
try:
worker_stream.send_multipart(message)
except Exception as e:
self.log_error(e)
queue_stream.on_recv(queue_message)
def requeue_message():
now = time() * 1000
for message in (item[1] for item in self.__queued_messages.itervalues() if item[0] + self.__message_timeouts.get(item[1][1], self.__retry_ms) < now):
queue_message(message)
requeue_callback = PeriodicCallback(requeue_message, self.__retry_ms, io_loop = self.__ioloop)
requeue_callback.start()
self.__ioloop.start()
self.__thread = None
self.__thread = Thread(target=loop)
self.__thread.daemon = True
self.__thread.start()
def stop(self):
if self.__ioloop:
self.__ioloop.stop()
def join(self):
if self.__thread:
self.__thread.join()
def enable_traceback_logging(self):
from new import instancemethod
from traceback import format_exc
def log_error(self, e):
logging.error(format_exc())
self.log_error = instancemethod(log_error, self)
@classmethod
def instance(cls):
#.........这里部分代码省略.........
示例5: WorkerConnection
# 需要导入模块: from zmq.eventloop.ioloop import IOLoop [as 别名]
# 或者: from zmq.eventloop.ioloop.IOLoop import start [as 别名]
class WorkerConnection(object):
'''Use a ``WorkerConnection`` to make RPCs to the remote worker service(s) or worker/router specified by ``address``.
``address`` may be either an enumerable of address strings or a string of comma separated addresses. RPC retries
and timeouts will happen by at most every ``abs(timeout)`` seconds when a periodic callback runs through all active
messages and checks for prolonged requests. This is also the default timeout for any new calls. ``timeout`` must not be
``0``.
Optionally pass any object or module with ``compress`` and ``decompress`` methods as the ``compression`` parameter to
compress messages. The module must implement the same algorithm used on the worker service. By default, messages are not
compressed.
Optionally pass any object or module with ``dumps`` and ``loads`` methods that convert an ``object`` to and from a
``str`` to replace the default ``cPickle`` serialization with a protocol of your choice.
Use ``auto_retry`` to specify whether or not messages should be retried by default. Retrying messages can cause substantial
congestion in your worker service. Use with caution.
'''
def __init__(self, address, timeout=10.0, compression=None, serialization=None, auto_retry=False):
if isinstance(address, str):
self.active_connections = {i.strip() for i in address.split(',')}
else:
self.active_connections = set(address)
self.message_address = 'inproc://WorkerConnection%s' % id(self)
self.__context = zmq.Context()
self.__queue_socket = self.__context.socket(zmq.PUSH)
self.__queue_socket.bind(self.message_address)
self.__thread = None
self.__timeout = timeout
self.__callbacks = {}
self.__queued_messages = {}
self.__message_auto_retry = {}
self.__message_timeouts = {}
self.__ioloop = None
self.__auto_retry = auto_retry
self.loads = serialization and serialization.loads or pickle.loads
self.dumps = serialization and serialization.dumps or pickle.dumps
self.compress = compression and compression.compress or (lambda x: x)
self.decompress = compression and compression.decompress or (lambda x: x)
def invoke(self, method, parameters={}, callback=None, timeout=0, auto_retry=None):
'''Invoke a ``method`` to be run on a remote worker process with the given ``parameters``. If specified, ``callback`` will be
invoked with any response from the remote worker. By default the worker will timeout or retry based on the settings of the
current ``WorkerConnection`` but ``timeout`` and ``auto_retry`` can be used for invocation specific behavior.
Note: ``callback`` will be invoked with ``{'error': 'timeout'}`` on ``timeout`` if ``auto_retry`` is false. Invocations
set to retry will never timeout and will instead be re-sent until a response is received. This behavior can be useful for
critical operations but has the potential to cause substantial congestion in the worker system. Use with caution. Negative
values of ``timeout`` will prevent messages from ever expiring or retrying regardless of ``auto_retry``. The default
values of ``timeout`` and ``auto_retry`` cause a fallback to the values used to initialize ``WorkerConnection``.
Alternativly, you can invoke methods with ``WorkerConnection.<module>.<method>(parameters, callback=None, timeout=0, auto_retry=None)``
where ``"<module>.<method>"`` will be passed as the ``method`` argument to ``invoke()``.
'''
self._queue_message(self.compress(self.dumps({'method': method, 'parameters': parameters})), callback, timeout, auto_retry)
def add_connection(self, address):
'''Connect to the worker at ``address``. Worker invocations will be round robin load balanced between all connected workers.'''
self._queue_message(address, command=WORKER_SOCKET_CONNECT)
def remove_connection(self, address):
'''Disconnect from the worker at ``address``. Worker invocations will be round robin load balanced between all connected workers.'''
self._queue_message(address, command=WORKER_SOCKET_DISCONNECT)
def set_connections(self, addresses):
'''A convenience method to set the connected addresses. A connection will be made to any new address included in the ``addresses``
enumerable and any currently connected address not included in ``addresses`` will be disconnected. If an address in ``addresses``
is already connected, it will not be affected.
'''
addresses = set(addresses)
to_remove = {a for a in self.active_connections if a not in addresses}
to_add = {a for a in addresses if a not in self.active_connections}
for a in to_remove:
self.remove_connection(a)
for a in to_add:
self.add_connection(a)
def __len__(self):
return len(self.__queued_messages)
def __getattr__(self, path):
return WorkerInvocation(path, self)
def _queue_message(self, message, callback=None, timeout=0, auto_retry=None, command=''):
if not self.__ioloop:
self.start()
message_id = str(uuid4())
if callback:
self.__callbacks[message_id] = callback
if timeout != 0:
self.__message_timeouts[message_id] = timeout
if auto_retry is not None:
self.__message_auto_retry[message_id] = auto_retry
self.__queue_socket.send_multipart((command, message_id, message))
def log_error(self, error):
logging.error(repr(error))
def start(self):
def loop():
#.........这里部分代码省略.........
示例6: IOPubThread
# 需要导入模块: from zmq.eventloop.ioloop import IOLoop [as 别名]
# 或者: from zmq.eventloop.ioloop.IOLoop import start [as 别名]
class IOPubThread(object):
"""An object for sending IOPub messages in a background thread
prevents a blocking main thread
IOPubThread(pub_socket).background_socket is a Socket-API-providing object
whose IO is always run in a thread.
"""
def __init__(self, socket, pipe=False):
self.socket = socket
self.background_socket = BackgroundSocket(self)
self._master_pid = os.getpid()
self._pipe_pid = os.getpid()
self._pipe_flag = pipe
self.io_loop = IOLoop()
if pipe:
self._setup_pipe_in()
self.thread = threading.Thread(target=self._thread_main)
self.thread.daemon = True
def _thread_main(self):
"""The inner loop that's actually run in a thread"""
self.io_loop.start()
self.io_loop.close()
def _setup_pipe_in(self):
"""setup listening pipe for subprocesses"""
ctx = self.socket.context
# use UUID to authenticate pipe messages
self._pipe_uuid = uuid.uuid4().bytes
pipe_in = ctx.socket(zmq.PULL)
pipe_in.linger = 0
try:
self._pipe_port = pipe_in.bind_to_random_port("tcp://127.0.0.1")
except zmq.ZMQError as e:
warn("Couldn't bind IOPub Pipe to 127.0.0.1: %s" % e +
"\nsubprocess output will be unavailable."
)
self._pipe_flag = False
pipe_in.close()
return
self._pipe_in = ZMQStream(pipe_in, self.io_loop)
self._pipe_in.on_recv(self._handle_pipe_msg)
def _handle_pipe_msg(self, msg):
"""handle a pipe message from a subprocess"""
if not self._pipe_flag or not self._is_master_process():
return
if msg[0] != self._pipe_uuid:
print("Bad pipe message: %s", msg, file=sys.__stderr__)
return
self.send_multipart(msg[1:])
def _setup_pipe_out(self):
# must be new context after fork
ctx = zmq.Context()
self._pipe_pid = os.getpid()
self._pipe_out = ctx.socket(zmq.PUSH)
self._pipe_out.connect("tcp://127.0.0.1:%i" % self._pipe_port)
def _is_master_process(self):
return os.getpid() == self._master_pid
def _have_pipe_out(self):
return os.getpid() == self._pipe_pid
def _check_mp_mode(self):
"""check for forks, and switch to zmq pipeline if necessary"""
if not self._pipe_flag or self._is_master_process():
return MASTER
else:
if not self._have_pipe_out():
# setup a new out pipe
self._setup_pipe_out()
return CHILD
def start(self):
"""Start the IOPub thread"""
self.thread.start()
def stop(self):
"""Stop the IOPub thread"""
self.io_loop.add_callback(self.io_loop.stop)
self.thread.join()
def close(self):
self.socket.close()
self.socket = None
@property
def closed(self):
return self.socket is None
def send_multipart(self, *args, **kwargs):
"""send_multipart schedules actual zmq send in my thread.
If my thread isn't running (e.g. forked process), send immediately.
#.........这里部分代码省略.........
示例7: IOPubThread
# 需要导入模块: from zmq.eventloop.ioloop import IOLoop [as 别名]
# 或者: from zmq.eventloop.ioloop.IOLoop import start [as 别名]
class IOPubThread(object):
"""An object for sending IOPub messages in a background thread
prevents a blocking main thread
IOPubThread(pub_socket).background_socket is a Socket-API-providing object
whose IO is always run in a thread.
"""
def __init__(self, socket, pipe=False):
self.socket = socket
self.background_socket = BackgroundSocket(self)
self._master_pid = os.getpid()
self._pipe_flag = pipe
self.io_loop = IOLoop()
if pipe:
self._setup_pipe_in()
self.thread = threading.Thread(target=self._thread_main)
self.thread.daemon = True
def _thread_main(self):
"""The inner loop that's actually run in a thread"""
self.io_loop.start()
self.io_loop.close()
def _setup_pipe_in(self):
"""setup listening pipe for subprocesses"""
ctx = self.socket.context
# use UUID to authenticate pipe messages
self._pipe_uuid = uuid.uuid4().bytes
pipe_in = ctx.socket(zmq.PULL)
pipe_in.linger = 0
try:
self._pipe_port = pipe_in.bind_to_random_port("tcp://127.0.0.1")
except zmq.ZMQError as e:
warnings.warn("Couldn't bind IOPub Pipe to 127.0.0.1: %s" % e +
"\nsubprocess output will be unavailable."
)
self._pipe_flag = False
pipe_in.close()
return
self._pipe_in = ZMQStream(pipe_in, self.io_loop)
self._pipe_in.on_recv(self._handle_pipe_msg)
def _handle_pipe_msg(self, msg):
"""handle a pipe message from a subprocess"""
if not self._pipe_flag or not self._is_master_process():
return
if msg[0] != self._pipe_uuid:
print("Bad pipe message: %s", msg, file=sys.__stderr__)
return
self.send_multipart(msg[1:])
def _setup_pipe_out(self):
# must be new context after fork
ctx = zmq.Context()
pipe_out = ctx.socket(zmq.PUSH)
pipe_out.linger = 3000 # 3s timeout for pipe_out sends before discarding the message
pipe_out.connect("tcp://127.0.0.1:%i" % self._pipe_port)
return ctx, pipe_out
def _is_master_process(self):
return os.getpid() == self._master_pid
def _check_mp_mode(self):
"""check for forks, and switch to zmq pipeline if necessary"""
if not self._pipe_flag or self._is_master_process():
return MASTER
else:
return CHILD
def start(self):
"""Start the IOPub thread"""
self.thread.start()
# make sure we don't prevent process exit
# I'm not sure why setting daemon=True above isn't enough, but it doesn't appear to be.
atexit.register(self.stop)
def stop(self):
"""Stop the IOPub thread"""
if not self.thread.is_alive():
return
self.io_loop.add_callback(self.io_loop.stop)
self.thread.join()
def close(self):
self.socket.close()
self.socket = None
@property
def closed(self):
return self.socket is None
def send_multipart(self, *args, **kwargs):
"""send_multipart schedules actual zmq send in my thread.
If my thread isn't running (e.g. forked process), send immediately.
"""
#.........这里部分代码省略.........