Python IOLoop.stop方法代码示例

# 需要导入模块: from zmq.eventloop.ioloop import IOLoop [as 别名]
# 或者: from zmq.eventloop.ioloop.IOLoop import stop [as 别名]
class AsyncServerAdapter(object):
    producer_class = Producer

    def __init__(self, backend_rep_uri, frontend_rep_uri, frontend_pub_uri,
                 control_pipe=None):
        self.uris = OrderedDict([
            ('backend_rep', backend_rep_uri),
            ('consumer_push_be', unique_ipc_uri()),
            ('consumer_pull_be', unique_ipc_uri()),
            ('frontend_rep_uri', frontend_rep_uri),
            ('frontend_pub_uri', frontend_pub_uri)
        ])
        self.control_pipe = control_pipe
        self.done = False
        logging.getLogger(log_label(self)).info("uris: %s", self.uris)

    def watchdog(self):
        if self.control_pipe is None:
            return
        elif not self.done and self.control_pipe.poll():
            self.done = True
            self.finish()

    def run(self):
        consumer = Process(target=Consumer(self.uris['backend_rep'],
                                        self.uris['consumer_push_be'],
                                        self.uris['consumer_pull_be']).run
        )
        producer = Process(target=self.producer_class(
                self.uris['frontend_rep_uri'],
                self.uris['frontend_pub_uri'],
                self.uris['consumer_pull_be'],
                self.uris['consumer_push_be']).run
        )
        self.io_loop = IOLoop()
        periodic_callback = PeriodicCallback(self.watchdog, 500, self.io_loop)
        periodic_callback.start()
        try:
            consumer.start()
            producer.start()
            self.io_loop.start()
        except KeyboardInterrupt:
            pass
        producer.terminate()
        consumer.terminate()
        logging.getLogger(log_label(self)).info('PRODUCER and CONSUMER have '
                                                'been terminated')

    def __del__(self):
        uris = [self.uris[label] for label in ('consumer_push_be',
                                               'consumer_pull_be', )]
        cleanup_ipc_uris(uris)

    def finish(self):
        logging.getLogger(log_label(self)).debug('"finish" request received')
        self.io_loop.stop()

# 需要导入模块: from zmq.eventloop.ioloop import IOLoop [as 别名]
# 或者: from zmq.eventloop.ioloop.IOLoop import stop [as 别名]
class WorkerConnection(object):

  def __init__(self, address, retry_ms=10000, compression=None, serialization=None):
    self.address = address
    self.message_address = 'inproc://WorkerConnection%s' % id(self)
    self.__context = zmq.Context()
    self.__queue_socket = self.__context.socket(zmq.PUSH)
    self.__queue_socket.bind(self.message_address)
    self.__thread = None
    self.__retry_ms = retry_ms
    self.__callbacks = {}
    self.__queued_messages = {}
    self.__message_timeouts = {}
    self.__ioloop = None
    self.loads = serialization and serialization.loads or pickle.loads
    self.dumps = serialization and serialization.dumps or pickle.dumps
    self.compress = compression and compression.compress or (lambda x: x)
    self.decompress = compression and compression.decompress or (lambda x: x)
  
  def invoke(self, method, parameters, callback=None, retry_ms=0):
    self._queue_message(self.compress(self.dumps({'method': method, 'parameters': parameters})), callback, retry_ms)
  
  def __len__(self):
    return len(self.__queued_messages)

  def __getattr__(self, path):
    return WorkerInvocation(path, self)

  def _queue_message(self, message, callback=None, retry_ms=0):
    if not self.__ioloop:
      self.start()
    message_id = str(uuid4())
    if callback:
      self.__callbacks[message_id] = callback
    if retry_ms > 0:
      self.__message_timeouts[message_id] = retry_ms
    self.__queue_socket.send_multipart(('', message_id, message))
  
  def log_error(self, error):
    logging.error(repr(error))

  def start(self):
    def loop():
      self.__ioloop = IOLoop()
      queue_socket = self.__context.socket(zmq.PULL)
      queue_socket.connect(self.message_address)
      queue_stream = ZMQStream(queue_socket, self.__ioloop)
      worker_socket = self.__context.socket(zmq.DEALER)
      worker_socket.connect(self.address)
      worker_stream = ZMQStream(worker_socket, self.__ioloop)

      def receive_response(message):
        self.__queued_messages.pop(message[1], None)
        self.__message_timeouts.pop(message[1], None)
        callback = self.__callbacks.pop(message[1], None)
        if callback:
          try:
            callback(self.loads(self.decompress(message[2])))
          except Exception as e:
            self.log_error(e)
      worker_stream.on_recv(receive_response)

      def queue_message(message):
        self.__queued_messages[message[1]] = (time() * 1000, message)
        try:
          worker_stream.send_multipart(message)
        except Exception as e:
          self.log_error(e)
      queue_stream.on_recv(queue_message)

      def requeue_message():
        now = time() * 1000
        for message in (item[1] for item in self.__queued_messages.itervalues() if item[0] + self.__message_timeouts.get(item[1][1], self.__retry_ms) < now):
          queue_message(message)
      requeue_callback = PeriodicCallback(requeue_message, self.__retry_ms, io_loop = self.__ioloop)
      requeue_callback.start()

      self.__ioloop.start()
      self.__thread = None
    self.__thread = Thread(target=loop)
    self.__thread.daemon = True
    self.__thread.start()

  def stop(self):
    if self.__ioloop:
      self.__ioloop.stop()
  
  def join(self):
    if self.__thread:
      self.__thread.join()

  def enable_traceback_logging(self):
    from new import instancemethod
    from traceback import format_exc
    def log_error(self, e):
      logging.error(format_exc())
    self.log_error = instancemethod(log_error, self)

  @classmethod
  def instance(cls):
#.........这里部分代码省略.........

# 需要导入模块: from zmq.eventloop.ioloop import IOLoop [as 别名]
# 或者: from zmq.eventloop.ioloop.IOLoop import stop [as 别名]

#.........这里部分代码省略.........
    return len(self.__queued_messages)

  def __getattr__(self, path):
    return WorkerInvocation(path, self)

  def _queue_message(self, message, callback=None, timeout=0, auto_retry=None, command=''):
    if not self.__ioloop:
      self.start()
    message_id = str(uuid4())
    if callback:
      self.__callbacks[message_id] = callback
    if timeout != 0:
      self.__message_timeouts[message_id] = timeout
    if auto_retry is not None:
      self.__message_auto_retry[message_id] = auto_retry
    self.__queue_socket.send_multipart((command, message_id, message))
  
  def log_error(self, error):
    logging.error(repr(error))

  def start(self):
    def loop():
      self.__ioloop = IOLoop()
      queue_socket = self.__context.socket(zmq.PULL)
      queue_socket.connect(self.message_address)
      queue_stream = ZMQStream(queue_socket, self.__ioloop)
      worker_socket = self.__context.socket(zmq.DEALER)
      for address in self.active_connections:
        worker_socket.connect(address)
      worker_stream = ZMQStream(worker_socket, self.__ioloop)

      def receive_response(message, response_override=None):
        self.__queued_messages.pop(message[1], None)
        self.__message_timeouts.pop(message[1], None)
        callback = self.__callbacks.pop(message[1], None)
        if callback:
          try:
            callback(response_override or self.loads(self.decompress(message[2])))
          except Exception as e:
            self.log_error(e)
            callback({'error': e})
      worker_stream.on_recv(receive_response)

      def queue_message(message):
        if message[0]:
          if message[0] == WORKER_SOCKET_CONNECT and message[2] not in self.active_connections:
            self.active_connections.add(message[2])
            worker_stream.socket.connect(message[2])
          elif message[0] == WORKER_SOCKET_DISCONNECT and message[2] in self.active_connections:
            self.active_connections.remove(message[2])
            worker_stream.socket.disconnect(message[2])
          return
        self.__queued_messages[message[1]] = (time(), message)
        try:
          worker_stream.send_multipart(message)
        except Exception as e:
          self.log_error(e)
      queue_stream.on_recv(queue_message)

      def timeout_message():
        now = time()
        for message, retry in [(item[1], self.__message_auto_retry.get(item[1][1], self.__auto_retry)) for item, t in ((i, self.__message_timeouts.get(i[1][1], self.__timeout)) for i in self.__queued_messages.itervalues()) if t >= 0 and (item[0] + t < now)]:
          if retry:
            logging.info('Worker timeout, requeuing ' + message[1])
            queue_message(message)
          else:
            receive_response(('', message[1]), {'error': 'timeout'})
      timeout_callback = PeriodicCallback(timeout_message, int(abs(self.__timeout * 1000.0)), io_loop = self.__ioloop)
      timeout_callback.start()

      self.__ioloop.start()
      self.__thread = None
    self.__thread = Thread(target=loop)
    self.__thread.daemon = True
    self.__thread.start()

  def stop(self):
    if self.__ioloop:
      self.__ioloop.stop()
  
  def join(self):
    if self.__thread:
      self.__thread.join()

  def enable_traceback_logging(self):
    from new import instancemethod
    from traceback import format_exc
    def log_error(self, e):
      logging.error(format_exc())
    self.log_error = instancemethod(log_error, self)

  @classmethod
  def instance(cls):
    '''Returns the default instance of ``WorkerConnection`` as configured by the options prefixed
      with ``worker_``, instantiating it if necessary. Import the ``workerconnection`` module within
      your ``TotoService`` and run it with ``--help`` to see all available options.
    '''
    if not hasattr(cls, '_instance'):
      cls._instance = cls(options.worker_address, timeout=options.worker_timeout, compression=options.worker_compression_module and __import__(options.worker_compression_module), serialization=options.worker_serialization_module and __import__(options.worker_serialization_module), auto_retry=options.worker_auto_retry)
    return cls._instance