Python Poller.unregister方法代码示例

# 需要导入模块: from zmq import Poller [as 别名]
# 或者: from zmq.Poller import unregister [as 别名]
    def __stop(self):
        """ try to stop all of this Role's services """

        # send commands
        poller = Poller()
        for (pipe, svc) in self.__services.items():
            pipe.send_string('STOP')
            self.logger.debug('sent STOP command to %s service' % svc)
            poller.register(pipe, POLLIN)

        # give services a few seconds to cleanup and exit before checking responses
        sleep(1)

        max_attempts = len(self.__services)
        attempts = 0

        while self.__some_alive() and attempts < max_attempts:
            attempts += 1

            # poll for any replies
            items = dict(poller.poll(60000))  # wait for messages

            # mark responding services as stopped
            alive = dict(self.__services)  # make copy
            for (pipe, svc) in alive.items():
                if pipe in items:
                    reply = pipe.recv_string()
                    if 'STOPPED' == reply:
                        self.logger.debug('received STOPPED control reply from %s service' % svc)
                        svc.join(timeout=5)  # STOPPED response should be sent right before svc exit
                        if svc.is_alive():
                            self.logger.error('%s service is still alive; not waiting' % svc)
                        else:
                            self.logger.debug('%s service thread stopped' % svc)
                        poller.unregister(pipe)
                        pipe.close()
                        del (self.__services[pipe])
                    else:
                        self.logger.debug('unknown control reply: %s' % reply)

            # log some useful info
            if len(self.__services) > 0:
                msg = '%s services still alive after %d cycles; ' % (
                    [str(s) for s in self.__services.values()], attempts)
                if attempts < max_attempts:
                    msg += 'waiting'
                else:
                    msg += 'giving up'
                self.logger.debug(msg)

# 需要导入模块: from zmq import Poller [as 别名]
# 或者: from zmq.Poller import unregister [as 别名]
class ZMQPoller(object):
    """A poller that can be used in the tornado IOLoop.
    
    This simply wraps a regular zmq.Poller, scaling the timeout
    by 1000, so that it is in seconds rather than milliseconds.
    """
    
    def __init__(self):
        self._poller = Poller()
    
    @staticmethod
    def _map_events(events):
        """translate IOLoop.READ/WRITE/ERROR event masks into zmq.POLLIN/OUT/ERR"""
        z_events = 0
        if events & IOLoop.READ:
            z_events |= POLLIN
        if events & IOLoop.WRITE:
            z_events |= POLLOUT
        if events & IOLoop.ERROR:
            z_events |= POLLERR
        return z_events
    
    @staticmethod
    def _remap_events(z_events):
        """translate zmq.POLLIN/OUT/ERR event masks into IOLoop.READ/WRITE/ERROR"""
        events = 0
        if z_events & POLLIN:
            events |= IOLoop.READ
        if z_events & POLLOUT:
            events |= IOLoop.WRITE
        if z_events & POLLERR:
            events |= IOLoop.ERROR
        return events
    
    def register(self, fd, events):
        return self._poller.register(fd, self._map_events(events))
    
    def modify(self, fd, events):
        return self._poller.modify(fd, self._map_events(events))
    
    def unregister(self, fd):
        return self._poller.unregister(fd)
    
    def poll(self, timeout):
        """poll in seconds rather than milliseconds.
        
        Event masks will be IOLoop.READ/WRITE/ERROR
        """
        z_events = self._poller.poll(1000*timeout)
        return [ (fd,self._remap_events(evt)) for (fd,evt) in z_events ]
    
    def close(self):
        pass

# 需要导入模块: from zmq import Poller [as 别名]
# 或者: from zmq.Poller import unregister [as 别名]
class ZmqSelector(BaseSelector):
    """A selector that can be used with asyncio's selector base event loops."""

    def __init__(self):
        # this maps file descriptors to keys
        self._fd_to_key = {}
        # read-only mapping returned by get_map()
        self._map = _SelectorMapping(self)
        self._poller = ZMQPoller()

    def _fileobj_lookup(self, fileobj):
        """Return a file descriptor from a file object.

        This wraps _fileobj_to_fd() to do an exhaustive search in case
        the object is invalid but we still have it in our map.  This
        is used by unregister() so we can unregister an object that
        was previously registered even if it is closed.  It is also
        used by _SelectorMapping.
        """
        try:
            return _fileobj_to_fd(fileobj)
        except ValueError:
            # Do an exhaustive search.
            for key in self._fd_to_key.values():
                if key.fileobj is fileobj:
                    return key.fd
            # Raise ValueError after all.
            raise

    def register(self, fileobj, events, data=None):
        if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)):
            raise ValueError("Invalid events: {!r}".format(events))

        key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data)

        if key.fd in self._fd_to_key:
            raise KeyError("{!r} (FD {}) is already registered"
                           .format(fileobj, key.fd))

        z_events = 0
        if events & EVENT_READ:
            z_events |= POLLIN
        if events & EVENT_WRITE:
            z_events |= POLLOUT
        try:
            self._poller.register(key.fd, z_events)
        except ZMQError as exc:
            raise OSError(exc.errno, exc.strerror) from exc

        self._fd_to_key[key.fd] = key
        return key

    def unregister(self, fileobj):
        try:
            key = self._fd_to_key.pop(self._fileobj_lookup(fileobj))
        except KeyError:
            raise KeyError("{!r} is not registered".format(fileobj)) from None
        try:
            self._poller.unregister(key.fd)
        except ZMQError as exc:
            self._fd_to_key[key.fd] = key
            raise OSError(exc.errno, exc.strerror) from exc
        return key

    def modify(self, fileobj, events, data=None):
        try:
            fd = self._fileobj_lookup(fileobj)
            key = self._fd_to_key[fd]
        except KeyError:
            raise KeyError("{!r} is not registered".format(fileobj)) from None
        if data == key.data and events == key.events:
            return key
        if events != key.events:
            z_events = 0
            if events & EVENT_READ:
                z_events |= POLLIN
            if events & EVENT_WRITE:
                z_events |= POLLOUT
            try:
                self._poller.modify(fd, z_events)
            except ZMQError as exc:
                raise OSError(exc.errno, exc.strerror) from exc

        key = key._replace(data=data, events=events)
        self._fd_to_key[key.fd] = key
        return key

    def close(self):
        self._fd_to_key.clear()
        self._poller = None

    def get_map(self):
        return self._map

    def _key_from_fd(self, fd):
        """Return the key associated to a given file descriptor.

        Parameters:
        fd -- file descriptor

#.........这里部分代码省略.........

# 需要导入模块: from zmq import Poller [as 别名]
# 或者: from zmq.Poller import unregister [as 别名]
class Subscriber(object):

    """Subscriber

    Subscribes to *addresses* for *topics*, and perform address translation of
    *translate* is true. The function *message_filter* can be used to
    discriminate some messages on the subscriber side. *topics* on the other
    hand performs filtering on the publishing side (from zeromq 3).

    Example::

        from posttroll.subscriber import Subscriber, get_pub_address

        addr = get_pub_address(service, timeout=2)
        sub = Subscriber([addr], 'my_topic')
        try:
            for msg in sub(timeout=2):
                print("Consumer got", msg)

        except KeyboardInterrupt:
            print("terminating consumer...")
            sub.close()

    """

    def __init__(self, addresses, topics='', message_filter=None,
                 translate=False):
        self._topics = self._magickfy_topics(topics)
        self._filter = message_filter
        self._translate = translate

        self.sub_addr = {}
        self.addr_sub = {}
        self.poller = None

        self._hooks = []
        self._hooks_cb = {}

        self.poller = Poller()
        self._lock = Lock()

        self.update(addresses)

        self._loop = True

    def add(self, address, topics=None):
        """Add *address* to the subscribing list for *topics*.

        It topics is None we will subscibe to already specified topics.
        """
        with self._lock:
            if address in self.addresses:
                return False

            topics = self._magickfy_topics(topics) or self._topics
            LOGGER.info("Subscriber adding address %s with topics %s",
                        str(address), str(topics))
            subscriber = get_context().socket(SUB)
            for t__ in topics:
                subscriber.setsockopt_string(SUBSCRIBE, six.text_type(t__))
            subscriber.connect(address)
            self.sub_addr[subscriber] = address
            self.addr_sub[address] = subscriber
            if self.poller:
                self.poller.register(subscriber, POLLIN)
            return True

    def remove(self, address):
        """Remove *address* from the subscribing list for *topics*.
        """
        with self._lock:
            try:
                subscriber = self.addr_sub[address]
            except KeyError:
                return False
            LOGGER.info("Subscriber removing address %s", str(address))
            if self.poller:
                self.poller.unregister(subscriber)
            del self.addr_sub[address]
            del self.sub_addr[subscriber]
            subscriber.close()
            return True

    def update(self, addresses):
        """Updating with a set of addresses.
        """
        if isinstance(addresses, six.string_types):
            addresses = [addresses, ]
        s0_, s1_ = set(self.addresses), set(addresses)
        sr_, sa_ = s0_.difference(s1_), s1_.difference(s0_)
        for a__ in sr_:
            self.remove(a__)
        for a__ in sa_:
            self.add(a__)
        return bool(sr_ or sa_)

    def add_hook_sub(self, address, topics, callback):
        """Specify a *callback* in the same stream (thread) as the main receive
        loop. The callback will be called with the received messages from the
        specified subscription.
#.........这里部分代码省略.........

# 需要导入模块: from zmq import Poller [as 别名]
# 或者: from zmq.Poller import unregister [as 别名]

#.........这里部分代码省略.........
    def register_input_sock(self, sock):
        """
        Register a given input socket as the ingest point for a Cornerstone
        instance.

        Keyward Arguments:
        sock - the input socket that is to be registered.

        Return: None

        Cornerstone does not support multiple input sockets, so any currently
        registered input socket will be discarded. This is a per instance
        limitation, in which case the primary concern is ipaddress collision.

        Example Usage:
        >>> from zmq import SUB, SUBSCRIBE
        >>> foo = Cornerstone()
        >>> ctx = foo.zmq_ctx
        >>> sock1 = ctx.socket(SUB)
        >>> sock1.connect('tcp://localhost:2880')
        >>> sock1.setsockopt(SUBSCRIBE, "")
        >>> assert foo._poll.sockets == {}
        >>> foo.register_input_sock(sock1)
        >>> assert foo._poll.sockets.has_key(sock1)
        >>> sock2 = ctx.socket(SUB)
        >>> sock2.connect('tcp://localhost:2881')
        >>> sock2.setsockopt(SUBSCRIBE, "")
        >>> foo.register_input_sock(sock2)
        >>> assert not foo._poll.sockets.has_key(sock1)
        >>> assert foo._poll.sockets.has_key(sock2)
        """
        # if there is an existing input socket, then it will be removed.
        if self._input_sock is not None:
            self._poll.unregister(self._input_sock)
            self._input_sock.close()
            self._input_sock = None

        self._input_sock = sock
        if self._input_sock is not None:
            self._poll.register(self._input_sock, POLLIN)


    def register_output_sock(self, sock):
        """
        Register a given output socket as the egress point for a Cornerstone
        instance.

        Keyward Arguments:
        sock - the output socket that is to be registered.

        Return: none

        Cornerstone does not support multiple output sockets,
        so any currently registered output socket will be discarded. This is
        a per instance limitation. In which case the primary concern is
        ipaddress collision.

        Example Usage:
        >>> from zmq import PUB
        >>> foo = Cornerstone()
        >>> ctx = foo.zmq_ctx
        >>> sock1 = ctx.socket(PUB)
        >>> sock1.bind('tcp://*:2880')
        >>> assert foo._output_sock == None
        >>> foo.register_output_sock(sock1)
        >>> assert foo._output_sock == sock1

# 需要导入模块: from zmq import Poller [as 别名]
# 或者: from zmq.Poller import unregister [as 别名]
class PushRequester(object):

    """Base requester class.
    """

    request_retries = 3

    def __init__(self, host, port):
        self._socket = None
        self._reqaddress = "tcp://" + host + ":" + str(port)
        self._poller = Poller()
        self._lock = Lock()
        self.failures = 0
        self.jammed = False

        self.connect()

    def connect(self):
        """Connect to the server
        """
        self._socket = context.socket(REQ)
        self._socket.connect(self._reqaddress)
        self._poller.register(self._socket, POLLIN)

    def stop(self):
        """Close the connection to the server
        """
        self._socket.setsockopt(LINGER, 0)
        self._socket.close()
        self._poller.unregister(self._socket)

    def reset_connection(self):
        """Reset the socket
        """
        self.stop()
        self.connect()

    def __del__(self, *args, **kwargs):
        self.stop()

    def send_and_recv(self, msg, timeout=REQ_TIMEOUT):

        with self._lock:
            retries_left = self.request_retries
            request = str(msg)
            self._socket.send(request)
            rep = None
            while retries_left:
                socks = dict(self._poller.poll(timeout))
                if socks.get(self._socket) == POLLIN:
                    reply = self._socket.recv()
                    if not reply:
                        LOGGER.error("Empty reply!")
                        break
                    rep = Message(rawstr=reply)
                    self.failures = 0
                    self.jammed = False
                    break
                else:
                    LOGGER.warning("Timeout from " + str(self._reqaddress)
                                   + ", retrying...")
                    # Socket is confused. Close and remove it.
                    self.stop()
                    retries_left -= 1
                    if retries_left <= 0:
                        LOGGER.error("Server doesn't answer, abandoning... " +
                                     str(self._reqaddress))
                        self.connect()
                        self.failures += 1
                        if self.failures == 5:
                            LOGGER.critical("Server jammed ? %s",
                                            self._reqaddress)
                            self.jammed = True
                        break
                    LOGGER.info("Reconnecting and resending " + str(msg))
                    # Create new connection
                    self.connect()
                    self._socket.send(request)

        return rep

# 需要导入模块: from zmq import Poller [as 别名]
# 或者: from zmq.Poller import unregister [as 别名]
class Distributor(Thread):
    def __init__(self, query):
        Thread.__init__(self)
        self._pool = ThreadPool(processes=4)
        self._workers = HashRing(WORKER_SERVERS)
        self._identity = bytes(uuid.uuid4())
        self._query = query
        self._init_sock()

    def _log(self, text):
        if LOG_DISTRIBUTOR:
            log_debug(self, text)

    def _init_sock(self):
        self._context = Context(1)
        self._poller = Poller()
        self._set_sock()

    def _set_sock(self):
        self._socket = self._context.socket(DEALER)
        self._socket.setsockopt(IDENTITY, self._identity)
        self._poller.register(self._socket, POLLIN)
        self._socket.connect(zmqaddr(self._get_broker(), BROKER_PORT))
        self._socket.send(PPP_READY)

    def _reset_sock(self):
        self._poller.unregister(self._socket)
        self._socket.setsockopt(LINGER, 0)
        self._socket.close()
        self._set_sock()

    def _get_broker(self):
        length = len(BROKER_SERVERS)
        return BROKER_SERVERS[randint(0, length - 1)]

    def _get_worker(self, uid):
        return self._workers.get_node(uid)

    def _get_user(self, buf):
        if len(buf) < USERNAME_SIZE:
            log_err(self, 'failed to get user, invalid length')
            raise Exception(log_get(self, 'failed to get user'))
        name = filter(lambda x:x != '*', buf[:USERNAME_SIZE])
        if not name:
            log_err(self, 'failed to get user')
            raise Exception(log_get(self, 'failed to get user'))
        return name

    def _get_token(self, buf):
        if len(buf) < UID_SIZE:
            log_err(self, 'failed to get token, invalid length')
            raise Exception(log_get(self, 'failed to get token'))
        uid = None
        token = None
        if buf[UID_SIZE - 1] == '*':
            user = self._get_user(buf)
            uid, token = self._query.user.get(user, 'uid', 'password')
        else:
            uid = buf[0:UID_SIZE]
            token = self._query.token.get(uid)
        if uid and token:
            return (uid, token)
        else:
            log_err(self, 'failed to get token')
            raise Exception(log_get(self, 'failed to get token'))

    def _reply(self, identity, seq, buf):
        msg = [identity, '', seq, buf]
        self._socket.send_multipart(msg)

    def _request(self, addr, **args):
        sock = io.connect(addr, WORKER_PORT)
        try:
            buf = bson.dumps(args)
            io.send_pkt(sock, buf)
            res = io.recv_pkt(sock)
            return unicode2str(bson.loads(res)['result'])
        finally:
            io.close(sock)

    def _proc(self, identity, seq, buf):
        uid, token = self._get_token(buf)
        if not uid or not token:
            log_err(self, 'failed to process, cannot get token')
            return
        addr = self._get_worker(uid)
        ret = self._request(addr, uid=uid, token=token, buf=buf)
        self._reply(identity, seq, ret)

    def _proc_safe(self, identity, seq, buf):
        try:
            self._proc(identity, seq, buf)
        except:
            log_err(self, 'failed to process')

    def _handler(self, identity, seq, buf):
        if DEBUG and not SAFE:
            self._proc(identity, seq, buf)
        else:
            self._proc_safe(identity, seq, buf)
#.........这里部分代码省略.........

# 需要导入模块: from zmq import Poller [as 别名]
# 或者: from zmq.Poller import unregister [as 别名]

#.........这里部分代码省略.........
                # Merge nodes list.
                new_nodes = self._nodes.update_nodes(nodes)
                # Create sockets for new nodes.
                for node_id, req_addr, pub_addr in new_nodes:
                    _LOG.debug("adding node: %s", node_id)
                    req_socket = self._open_req_socket(req_addr)
                    sub_socket = self._open_sub_socket(pub_addr)
                    added.append((node_id, (req_socket, sub_socket)))
        self._nodes_sockets.update(dict(added))  # save sockets
        return len(added) > 0  # nodes list got changed?

    def _add_node(self, node_id, req_addr, pub_addr, req_socket=None):
        """Add one node to the list and create the sockets for the request
        and service URLs (subscribe to the node).
        """
        _LOG.debug("adding node: %s", node_id)
        # Add node to the nodes list.
        self._nodes.add_node(node_id, req_addr, pub_addr)
        if req_socket is None:  # does that not yet exist?
            req_socket = self._open_req_socket(req_addr)
        sub_socket = self._open_sub_socket(pub_addr)
        # Remember sockets.
        self._nodes_sockets[node_id] = (req_socket, sub_socket)

    def _remove_nodes(self, removed_nodes):
        """Unregister and remove a node, remove all pending request, too.
        """
        for node_id in removed_nodes:
            _LOG.debug("removing node: %s", node_id)
            # Unregister sockets from poller.
            req_socket, pub_socket = self._nodes_sockets.get(
                node_id, (None, None))
            if req_socket is not None:
                self._poller.unregister(req_socket)
            if pub_socket is not None:
                self._poller.unregister(pub_socket)
            # Forget the sockets and erase request queue.
            self._nodes_sockets.pop(node_id, None)
            self._nodes_requests.pop(node_id, None)
        return len(removed_nodes) > 0  # nodes list got changed?

    def _handle_responses(self, poll_sockets):
        """When a request from the queue gets an answer call the function for
        that request, if possible.
        """
        for node_id, (req_socket, _) in self._nodes_sockets.items():
            # Some node did replay?
            if req_socket in poll_sockets:
                # Get the answer and call the callback function with it, if
                # there was one.
                response = req_socket.recv_json()
                requests = self._nodes_requests[node_id]
                if requests:
                    in_progress, _, callback = requests[0]
                    if in_progress:
                        callback(response)
                        # Request done, remove it from the queue.
                        self._nodes_requests[node_id] = requests[1:]

    def _handle_request(self, node_changes, action, data):
        # Handle the requested action.
        _LOG.debug("Request %s: %s", action, str(data))
        if action == self.CMD_NEW:  # new node, register it
            node_id, req_addr, pub_addr = data
            self._add_node(node_id, req_addr, pub_addr)
            node_changes[0] = True  # a node was added!

# 需要导入模块: from zmq import Poller [as 别名]
# 或者: from zmq.Poller import unregister [as 别名]
class Subscriber(object):
    def __init__(self, addresses, translate=False):
        self._addresses = addresses
        self._translate = translate
        self.subscribers = []
        self._poller = Poller()
        for addr in self._addresses:
            subscriber = context.socket(SUB)
            subscriber.setsockopt(SUBSCRIBE, "pytroll")
            subscriber.connect(addr)
            self.subscribers.append(subscriber)
            self._poller.register(subscriber)
        self._lock = Lock()
        self._loop = True

    @property
    def sub_addr(self):
        return dict(zip(self.subscribers, self._addresses))

    @property
    def addr_sub(self):
        return dict(zip(self._addresses, self.subscribers))

    def reset(self, addr):
        with self._lock:
            idx = self._addresses.index(addr)
            self._poller.unregister(self.subscribers[idx])
            self.subscribers[idx].setsockopt(LINGER, 0)
            self.subscribers[idx].close()
            self.subscribers[idx] = context.socket(SUB)
            self.subscribers[idx].setsockopt(SUBSCRIBE, "pytroll")
            self.subscribers[idx].connect(addr)
            self._poller.register(self.subscribers[idx], POLLIN)

    def recv(self, timeout=None):
        """Receive a message, timeout in seconds.
        """

        if timeout:
            timeout *= 1000

        while self._loop:
            with self._lock:
                if not self._loop:
                    break
                subs = dict(self._poller.poll(timeout=timeout))
                if subs:
                    for sub in self.subscribers:
                        if sub in subs and subs[sub] == POLLIN:
                            msg = Message.decode(sub.recv())
                            if self._translate:
                                url = urlsplit(self.sub_addr[sub])
                                host = url[1].split(":")[0]
                                msg.sender = msg.sender.split("@")[0] + "@" + host
                            yield msg
                else:
                    yield None

    def stop(self):
        """Stop the subscriber
        """
        with self._lock:
            self._loop = False
            for sub in self.subscribers:
                self._poller.unregister(sub)
                sub.setsockopt(LINGER, 0)
                sub.close()

# 需要导入模块: from zmq import Poller [as 别名]
# 或者: from zmq.Poller import unregister [as 别名]
class JobContext(object):
    def __init__(self, job_id):
        self.job_id = job_id
        self.connections_by_name = {}
        self.connections_by_socket = {}
        self.poller = Poller()
        self.num_pending_reqs = 0

    def send_req(self, worker_node, msg, callback):
        """Send a message and register callback for response. Message
        should be unserialized json
        """
        node_name = worker_node["name"]
        if self.connections_by_name.has_key(node_name):
            conn = self.connections_by_name[node_name]
            assert conn.port==worker_node["worker_port"]
            assert conn.contact_address==worker_node["contact_address"]
        else:
            conn = Connection(node_name, worker_node["contact_address"],
                              worker_node["worker_port"])
            conn.connect()
            self.connections_by_name[node_name] = conn
            self.connections_by_socket[conn.socket] = conn
            self.poller.register(conn.socket, flags=POLLIN)
        self.num_pending_reqs += 1
        conn.send(json.dumps(msg), callback)
        if msg.has_key("mtype"): mtype = msg["mtype"]
        else: mtype = "unknown"
        logger.debug("Sent message of type %s to %s" %
                     (mtype, worker_node["name"]))

    def poll(self, timeout_in_ms):
        events = self.poller.poll(timeout_in_ms)
        if len(events)==0:
            return 0
        for (socket, num) in events:
            if not self.connections_by_socket.has_key(socket):
                raise ZmqServerError("Poll: Could not find registered socket for socket %s" % socket)
            conn = self.connections_by_socket[socket]
            if not conn.callback:
                raise ZmqServerError("Received event for connection %s when no request pending" % conn.node_name)
            msg = socket.recv()
            logger.debug("message received from worker %s" % conn.node_name)
            callback = conn.callback
            # We clear the callback field of the connection just before we call
            # the callback. This lets us disconnect within the callback,
            # if we want to (e.g. after receiving a stop ack).
            conn.callback = None
            callback(msg)
            self.num_pending_reqs = self.num_pending_reqs - 1
        return len(events)

    def disconnect(self, node_name):
        if not self.connections_by_name.has_key(node_name):
            raise ZmqServerError("Attempting to disconnect from %s but no conection found" % node_name)
        conn = self.connections_by_name[node_name]
        assert conn.callback==None
        del self.connections_by_name[node_name]
        del self.connections_by_socket[conn.socket]
        self.poller.unregister(conn.socket)
        conn.socket.close()

    def stop_job(self):
        node_names = [n for n in self.connections_by_name.keys()]
        for node_name in node_names:
            self.disconnect(node_name)