Python Queue.get方法代码示例

# 需要导入模块: from eventlet.queue import Queue [as 别名]
# 或者: from eventlet.queue.Queue import get [as 别名]
    def _make_app_iter(self, node, source):
        """
        Returns an iterator over the contents of the source (via its read
        func).  There is also quite a bit of cleanup to ensure garbage
        collection works and the underlying socket of the source is closed.

        :param source: The httplib.Response object this iterator should read
                       from.
        :param node: The node the source is reading from, for logging purposes.
        """
        try:
            # Spawn reader to read from the source and place in the queue.
            # We then drop any reference to the source or node, for garbage
            # collection purposes.
            queue = Queue(1)
            spawn_n(self._make_app_iter_reader, node, source, queue,
                    self.app.logger.thread_locals)
            source = node = None
            while True:
                chunk = queue.get(timeout=self.app.node_timeout)
                if isinstance(chunk, bool):  # terminator
                    success = chunk
                    if not success:
                        raise Exception(_('Failed to read all data'
                                          ' from the source'))
                    break
                yield chunk
        except Empty:
            raise ChunkReadTimeout()
        except (GeneratorExit, Timeout):
            self.app.logger.warn(_('Client disconnected on read'))
        except Exception:
            self.app.logger.exception(_('Trying to send to client'))
            raise

# 需要导入模块: from eventlet.queue import Queue [as 别名]
# 或者: from eventlet.queue.Queue import get [as 别名]
class LocalMailbox(Mailbox):

    def __init__(self):
        self._queue = Queue()

    def put(self, message):
        self._queue.put(message)

    def get(self):
        return self._queue.get()

# 需要导入模块: from eventlet.queue import Queue [as 别名]
# 或者: from eventlet.queue.Queue import get [as 别名]
class Actor(ActorBase):
    def __init__(self, callback):
        self._inbox = Queue()
        self._callback = callback
        self._greenlet = None

    def run(self, *args, **kwargs):
        greenlet_id = id(eventlet.getcurrent())
        _actor_map[greenlet_id] = self
        try:
            self._callback(*args, **kwargs)
        finally:
            del _actor_map[greenlet_id]

    def spawn(self, *args, **kwargs):
        self._greenlet = _actor_pool.spawn(self.run, *args, **kwargs)

    def link(self, func, *args, **kwargs):
        if self._greenlet is None:
            return
        return self._greenlet.link(func, *args, **kwargs)

    def unlink(self, func, *args, **kwargs):
        if self._greenlet is None:
            return
        return self._greenlet.unlink(func, *args, **kwargs)

    def cancel(self, *throw_args):
        if self._greenlet is None:
            return
        return self._greenlet.cancel(*throw_args)

    def kill(self, *throw_args):
        if self._greenlet is None:
            return
        return self._greenlet.kill(*throw_args)

    def wait(self):
        if self._greenlet is None:
            return
        return self._greenlet.wait()

    def send(self, message):
        self._inbox.put(message)

    def receive(self):
        return self._inbox.get()

# 需要导入模块: from eventlet.queue import Queue [as 别名]
# 或者: from eventlet.queue.Queue import get [as 别名]
class LocalMailbox(Mailbox):

    def __init__(self):
        self._queue = Queue()

    def put(self, message):
        self._queue.put(message)

    def get(self):
        return self._queue.get()

    def encode(self):
        raise NotImplementedError

    @staticmethod
    def decode(params):
        raise NotImplementedError

# 需要导入模块: from eventlet.queue import Queue [as 别名]
# 或者: from eventlet.queue.Queue import get [as 别名]
class DownloadPool(object):

    def __init__(self, settings):
        self.temp_dir = settings['temp_dir']
        self.download_path = settings['download_path']
        self.connection_pool = Queue(settings['connections'])

        for _ in xrange(settings['connections']):
            self.connection_pool.put(NNTP(settings['host'], settings['port'], settings['username'], settings['password']))

    def download(self, segment):
        #print 'getting', segment['segment']
        # Get an availble connection; if there are none, block until available.
        connection = self.connection_pool.get()
        segment_path = connection.get_body(segment['segment'], self.temp_dir)
        # Connection is done, put it back in the ready queue.
        self.connection_pool.put(connection)
        #print 'got', segment_path
        Tracker.downloaded += segment['segment_bytes']
        #print Tracker.downloaded
        return segment_path

# 需要导入模块: from eventlet.queue import Queue [as 别名]
# 或者: from eventlet.queue.Queue import get [as 别名]
class EventletConnection(Connection):
    """
    An implementation of :class:`.Connection` that utilizes ``eventlet``.
    """

    _total_reqd_bytes = 0
    _read_watcher = None
    _write_watcher = None

    _socket_impl = eventlet.green.socket
    _ssl_impl = eventlet.green.ssl

    @classmethod
    def initialize_reactor(cls):
        eventlet.monkey_patch()

    def __init__(self, *args, **kwargs):
        Connection.__init__(self, *args, **kwargs)

        self.connected_event = Event()
        self._write_queue = Queue()

        self._callbacks = {}
        self._push_watchers = defaultdict(set)

        self._connect_socket()

        self._read_watcher = eventlet.spawn(lambda: self.handle_read())
        self._write_watcher = eventlet.spawn(lambda: self.handle_write())
        self._send_options_message()

    def close(self):
        with self.lock:
            if self.is_closed:
                return
            self.is_closed = True

        log.debug("Closing connection (%s) to %s" % (id(self), self.host))

        cur_gthread = eventlet.getcurrent()

        if self._read_watcher and self._read_watcher != cur_gthread:
            self._read_watcher.kill()
        if self._write_watcher and self._write_watcher != cur_gthread:
            self._write_watcher.kill()
        if self._socket:
            self._socket.close()
        log.debug("Closed socket to %s" % (self.host,))

        if not self.is_defunct:
            self.error_all_callbacks(
                ConnectionShutdown("Connection to %s was closed" % self.host))
            # don't leave in-progress operations hanging
            self.connected_event.set()

    def handle_close(self):
        log.debug("connection closed by server")
        self.close()

    def handle_write(self):
        while True:
            try:
                next_msg = self._write_queue.get()
                self._socket.sendall(next_msg)
            except socket.error as err:
                log.debug("Exception during socket send for %s: %s", self, err)
                self.defunct(err)
                return  # Leave the write loop

    def handle_read(self):
        run_select = partial(select.select, (self._socket,), (), ())
        while True:
            try:
                run_select()
            except Exception as exc:
                if not self.is_closed:
                    log.debug("Exception during read select() for %s: %s",
                              self, exc)
                    self.defunct(exc)
                return

            try:
                buf = self._socket.recv(self.in_buffer_size)
                self._iobuf.write(buf)
            except socket.error as err:
                if not is_timeout(err):
                    log.debug("Exception during socket recv for %s: %s",
                              self, err)
                    self.defunct(err)
                    return  # leave the read loop

            if self._iobuf.tell():
                self.process_io_buffer()
            else:
                log.debug("Connection %s closed by server", self)
                self.close()
                return

    def push(self, data):
        chunk_size = self.out_buffer_size
#.........这里部分代码省略.........

# 需要导入模块: from eventlet.queue import Queue [as 别名]
# 或者: from eventlet.queue.Queue import get [as 别名]
class Client(baseasync.BaseAsync):
    def __init__(self, *args, **kwargs):
        super(Client, self).__init__(*args, **kwargs)
        self.pool = eventlet.greenpool.GreenPool(DEFAULT_POOL_SIZE)
        self.reader_thread = None
        self.writer_thread = None
        self.queue = Queue(DEFAULT_MAX_QUEUE_SIZE)
        self.max_pending = MAX_PENDING
        self.closing = False

    def build_socket(self, family=socket.AF_INET):
        return socket.socket(family)

    def wrap_secure_socket(self, s, ssl_version):
        return GreenSSLSocket(s, ssl_version=ssl_version)

    def connect(self):
        super(Client, self).connect()
        self.closing = False
        self.reader_thread = eventlet.greenthread.spawn(self._reader_run)
        self.writer_thread = eventlet.greenthread.spawn(self._writer_run)

    def dispatch(self, fn, *args, **kwargs):
        if LOG.isEnabledFor(logging.DEBUG):
            LOG.debug("Dispatching: Pending {0}".format(len(self._pending)))
        self.pool.spawn_n(fn, *args, **kwargs)

    def shutdown(self):
        self.closing = True
        if len(self._pending) + self.queue.qsize() == 0:
            self._end_close()

    def close(self):
        self.shutdown()
        self.wait()

    def _end_close(self):
        self.writer_thread.kill()
        self.reader_thread.kill()

        super(Client, self).close()

        self.writer_thread = None
        self.reader_thread = None

    def sendAsync(self, header, value, onSuccess, onError, no_ack=False):
        if self.closing:
            raise common.ConnectionClosed("Client is closing, can't queue more operations.")

        if self.faulted:
            self._raise(common.ConnectionFaulted("Can't send message when connection is on a faulted state."), onError)
            return  # skip the rest

        # fail fast on NotConnected
        if not self.isConnected:
            self._raise(common.NotConnected("Not connected."), onError)
            return  # skip the rest

        if LOG.isEnabledFor(logging.DEBUG):
            LOG.debug("Queue: {0}".format(self.queue.qsize()))
        self.queue.put((header, value, onSuccess, onError, no_ack))
        eventlet.sleep(0)

    def wait(self):
        self.queue.join()

    def send(self, header, value):
        done = eventlet.event.Event()

        class Dummy:
            pass

        d = Dummy()
        d.error = None
        d.result = None

        def innerSuccess(m, r, value):
            d.result = (m, r, value)
            done.send()

        def innerError(e):
            d.error = e
            done.send()

        self.sendAsync(header, value, innerSuccess, innerError)

        done.wait()  # TODO(Nacho): should be add a default timeout?
        if d.error:
            raise d.error
        return d.result

    def _writer_run(self):
        while self.isConnected and not self.faulted:
            try:
                while len(self._pending) > self.max_pending:
                    eventlet.sleep(0)
                (header, value, onSuccess, onError, no_ack) = self.queue.get()
                super(Client, self).sendAsync(header, value, onSuccess, onError, no_ack)
            except common.ConnectionFaulted:
                pass
#.........这里部分代码省略.........

# 需要导入模块: from eventlet.queue import Queue [as 别名]
# 或者: from eventlet.queue.Queue import get [as 别名]
class EventletConnection(Connection):
    """
    An implementation of :class:`.Connection` that utilizes ``eventlet``.

    This implementation assumes all eventlet monkey patching is active. It is not tested with partial patching.
    """

    _read_watcher = None
    _write_watcher = None

    _socket_impl = eventlet.green.socket
    _ssl_impl = eventlet.green.ssl

    _timers = None
    _timeout_watcher = None
    _new_timer = None

    @classmethod
    def initialize_reactor(cls):
        eventlet.monkey_patch()
        if not cls._timers:
            cls._timers = TimerManager()
            cls._timeout_watcher = eventlet.spawn(cls.service_timeouts)
            cls._new_timer = Event()

    @classmethod
    def create_timer(cls, timeout, callback):
        timer = Timer(timeout, callback)
        cls._timers.add_timer(timer)
        cls._new_timer.set()
        return timer

    @classmethod
    def service_timeouts(cls):
        """
        cls._timeout_watcher runs in this loop forever.
        It is usually waiting for the next timeout on the cls._new_timer Event.
        When new timers are added, that event is set so that the watcher can
        wake up and possibly set an earlier timeout.
        """
        timer_manager = cls._timers
        while True:
            next_end = timer_manager.service_timeouts()
            sleep_time = max(next_end - time.time(), 0) if next_end else 10000
            cls._new_timer.wait(sleep_time)
            cls._new_timer.clear()

    def __init__(self, *args, **kwargs):
        Connection.__init__(self, *args, **kwargs)

        self._write_queue = Queue()

        self._connect_socket()

        self._read_watcher = eventlet.spawn(lambda: self.handle_read())
        self._write_watcher = eventlet.spawn(lambda: self.handle_write())
        self._send_options_message()

    def close(self):
        with self.lock:
            if self.is_closed:
                return
            self.is_closed = True

        log.debug("Closing connection (%s) to %s" % (id(self), self.host))

        cur_gthread = eventlet.getcurrent()

        if self._read_watcher and self._read_watcher != cur_gthread:
            self._read_watcher.kill()
        if self._write_watcher and self._write_watcher != cur_gthread:
            self._write_watcher.kill()
        if self._socket:
            self._socket.close()
        log.debug("Closed socket to %s" % (self.host,))

        if not self.is_defunct:
            self.error_all_requests(
                ConnectionShutdown("Connection to %s was closed" % self.host))
            # don't leave in-progress operations hanging
            self.connected_event.set()

    def handle_close(self):
        log.debug("connection closed by server")
        self.close()

    def handle_write(self):
        while True:
            try:
                next_msg = self._write_queue.get()
                self._socket.sendall(next_msg)
            except socket.error as err:
                log.debug("Exception during socket send for %s: %s", self, err)
                self.defunct(err)
                return  # Leave the write loop

    def handle_read(self):
        run_select = partial(select.select, (self._socket,), (), ())
        while True:
            try:
#.........这里部分代码省略.........

# 需要导入模块: from eventlet.queue import Queue [as 别名]
# 或者: from eventlet.queue.Queue import get [as 别名]
class EventletConnection(Connection):
    """
    An implementation of :class:`.Connection` that utilizes ``eventlet``.
    """

    _total_reqd_bytes = 0
    _read_watcher = None
    _write_watcher = None
    _socket = None

    @classmethod
    def factory(cls, *args, **kwargs):
        timeout = kwargs.pop('timeout', 5.0)
        conn = cls(*args, **kwargs)
        conn.connected_event.wait(timeout)
        if conn.last_error:
            raise conn.last_error
        elif not conn.connected_event.is_set():
            conn.close()
            raise OperationTimedOut("Timed out creating connection")
        else:
            return conn

    def __init__(self, *args, **kwargs):
        Connection.__init__(self, *args, **kwargs)

        self.connected_event = Event()
        self._iobuf = StringIO()
        self._write_queue = Queue()

        self._callbacks = {}
        self._push_watchers = defaultdict(set)

        self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self._socket.settimeout(1.0)
        self._socket.connect((self.host, self.port))

        if self.sockopts:
            for args in self.sockopts:
                self._socket.setsockopt(*args)

        self._read_watcher = eventlet.spawn(lambda: self.handle_read())
        self._write_watcher = eventlet.spawn(lambda: self.handle_write())
        self._send_options_message()

    def close(self):
        with self.lock:
            if self.is_closed:
                return
            self.is_closed = True

        log.debug("Closing connection (%s) to %s" % (id(self), self.host))

        cur_gthread = eventlet.getcurrent()

        if self._read_watcher and self._read_watcher != cur_gthread:
            self._read_watcher.kill()
        if self._write_watcher and self._write_watcher != cur_gthread:
            self._write_watcher.kill()
        if self._socket:
            self._socket.close()
        log.debug("Closed socket to %s" % (self.host,))

        if not self.is_defunct:
            self.error_all_callbacks(
                ConnectionShutdown("Connection to %s was closed" % self.host))
            # don't leave in-progress operations hanging
            self.connected_event.set()

    def handle_write(self):
        while True:
            try:
                next_msg = self._write_queue.get()
                self._socket.send(next_msg)
            except socket.error as err:
                log.debug(
                    "Exception during socket sendall for %s: %s", self, err)
                self.defunct(err)
                return  # Leave the write loop

    def handle_read(self):
        run_select = partial(select.select, (self._socket,), (), ())
        while True:
            try:
                run_select()
            except Exception as exc:
                if not self.is_closed:
                    log.debug(
                        "Exception during read select() for %s: %s", self, exc)
                    self.defunct(exc)
                return

            try:
                buf = self._socket.recv(self.in_buffer_size)
                self._iobuf.write(buf)
            except socket.error as err:
                if not is_timeout(err):
                    log.debug(
                        "Exception during socket recv for %s: %s", self, err)
                    self.defunct(err)
#.........这里部分代码省略.........

# 需要导入模块: from eventlet.queue import Queue [as 别名]
# 或者: from eventlet.queue.Queue import get [as 别名]
class BaseNode(object):
    SIZE = 256
    INPUT_SPEED = 512
    OUTPUT_SPEED = 512
    BUFFER_SIZE = 10
    BITRATE = 256

    def __init__(self, id_):
        self.id = id_
        self._peers = {}
        self.peers_info = {}
        self.available_peers = []
        self.main_channel = PriorityQueue()
        self.data_channel = Queue(1)
        self.sending_queue = Queue()
        self.receiving_queue = Queue()
        self.buffer = {}
        # for stats
        self.sent_bytes = 0
        self.received_bytes = 0
        self.delays = {}

    def __repr__(self):
        return "Node(id=%d, peers=%s)" % (self.id, self.peers_info.keys())

    @property
    def peers(self):
        return self._peers

    @peers.setter
    def peers(self, peers):
        self._peers = peers
        self.available_peers = peers.keys()
        self.peers_info = dict((peer_id, []) for peer_id in peers.keys())

    def run(self):
        return eventlet.spawn(self._do_main_loop)

    def _do_main_loop(self):
        sending_thread = eventlet.spawn(self._sending, self.sending_queue, self.main_channel)
        receiving_thread = eventlet.spawn(self._receiving, self.receiving_queue, self.main_channel)
        start_time = time.time()
        while True:
            message = self.main_channel.get()
            block_id = message.block_id
            peer_id = message.sender_id
            log.info("%s: %s" % (self, message))
            if message.type == MessageType.notify:
                self._do_receive_notify(peer_id, block_id)
            elif message.type == MessageType.request:
                self._do_receive_request(peer_id, block_id)
            elif message.type == MessageType.done_sending:
                self._after_sending(peer_id, block_id)
            elif message.type == MessageType.done_receiving:
                self._after_receiving(peer_id, block_id, message.payload[0])
            elif message.type == MessageType.exit:
                self._broadcast(block_id, MessageType.exit)
                sending_thread.kill()
                receiving_thread.kill()
                break
            self._try_to_request()
            eventlet.sleep(0)
            log.debug("%s: available peers - %s" % (self, self.available_peers))
        transmission_time = time.time() - start_time

        stats = {
            'input_load': round(self.received_bytes / transmission_time / self.INPUT_SPEED, 4),
            'output_load': round(self.sent_bytes / transmission_time / self.OUTPUT_SPEED, 4),
            'delays': self.delays,
        }
        return stats

    def _sending(self, queue, back_queue):
        while True:
            block_id, receiver_id, part = queue.get()
            receiver = self.peers[receiver_id]
            if receiver.data_channel.full():
                queue.put((block_id, receiver.id, part))
                eventlet.sleep(0)
                continue
            # log.debug("Send %d/%d of %d to %d" %(part+1, self.SIZE, block_id, receiver_id))
            receiver.data_channel.put((block_id, self.id, part))
            if part < self.SIZE - 1:
                queue.put((block_id, receiver_id, part + 1))
            else:
                self.sent_bytes += self.SIZE
                back_queue.put(Message(MessageType.done_sending, receiver_id, block_id))
            eventlet.sleep(1/self.OUTPUT_SPEED)

    def _receiving(self, queue, back_queue):
        while True:
            block_id, sender_id, part = self.data_channel.get()
            # log.debug("%s: Get %d/%d of %d to %d" % (self, part+1, self.SIZE, block_id, sender_id))
            if part == self.SIZE - 1:
                self.received_bytes += self.SIZE
                back_queue.put(Message(MessageType.done_receiving, sender_id, block_id, block_id))
            eventlet.sleep(1/self.INPUT_SPEED)

    def _after_receiving(self, sender_id, block_id, block):
        # if len(self.buffer) == self.BUFFER_SIZE:
#.........这里部分代码省略.........

# 需要导入模块: from eventlet.queue import Queue [as 别名]
# 或者: from eventlet.queue.Queue import get [as 别名]
class EventletConnection(Connection):
    """
    An implementation of :class:`.Connection` that utilizes ``eventlet``.
    """

    _total_reqd_bytes = 0
    _read_watcher = None
    _write_watcher = None
    _socket = None

    @classmethod
    def factory(cls, *args, **kwargs):
        timeout = kwargs.pop('timeout', 5.0)
        conn = cls(*args, **kwargs)
        conn.connected_event.wait(timeout)
        if conn.last_error:
            raise conn.last_error
        elif not conn.connected_event.is_set():
            conn.close()
            raise OperationTimedOut("Timed out creating connection")
        else:
            return conn

    def __init__(self, *args, **kwargs):
        Connection.__init__(self, *args, **kwargs)

        self.connected_event = Event()
        self._write_queue = Queue()

        self._callbacks = {}
        self._push_watchers = defaultdict(set)

        sockerr = None
        addresses = socket.getaddrinfo(
            self.host, self.port, socket.AF_UNSPEC, socket.SOCK_STREAM
        )
        for (af, socktype, proto, canonname, sockaddr) in addresses:
            try:
                self._socket = socket.socket(af, socktype, proto)
                self._socket.settimeout(1.0)
                self._socket.connect(sockaddr)
                sockerr = None
                break
            except socket.error as err:
                sockerr = err
        if sockerr:
            raise socket.error(
                sockerr.errno,
                "Tried connecting to %s. Last error: %s" % (
                    [a[4] for a in addresses], sockerr.strerror)
            )

        if self.sockopts:
            for args in self.sockopts:
                self._socket.setsockopt(*args)

        self._read_watcher = eventlet.spawn(lambda: self.handle_read())
        self._write_watcher = eventlet.spawn(lambda: self.handle_write())
        self._send_options_message()

    def close(self):
        with self.lock:
            if self.is_closed:
                return
            self.is_closed = True

        log.debug("Closing connection (%s) to %s" % (id(self), self.host))

        cur_gthread = eventlet.getcurrent()

        if self._read_watcher and self._read_watcher != cur_gthread:
            self._read_watcher.kill()
        if self._write_watcher and self._write_watcher != cur_gthread:
            self._write_watcher.kill()
        if self._socket:
            self._socket.close()
        log.debug("Closed socket to %s" % (self.host,))

        if not self.is_defunct:
            self.error_all_callbacks(
                ConnectionShutdown("Connection to %s was closed" % self.host))
            # don't leave in-progress operations hanging
            self.connected_event.set()

    def handle_close(self):
        log.debug("connection closed by server")
        self.close()

    def handle_write(self):
        while True:
            try:
                next_msg = self._write_queue.get()
                self._socket.send(next_msg)
            except socket.error as err:
                log.debug("Exception during socket send for %s: %s", self, err)
                self.defunct(err)
                return  # Leave the write loop

    def handle_read(self):
        run_select = partial(select.select, (self._socket,), (), ())
#.........这里部分代码省略.........

# 需要导入模块: from eventlet.queue import Queue [as 别名]
# 或者: from eventlet.queue.Queue import get [as 别名]
class RelayServer(object):
    
    def __init__(self, hostname, port):
        self._address = (hostname, port)
        self._clients = {}
        self._messages = []
        self._pool = eventlet.GreenPool(256)
        self._queue = Queue()

    def start(self):
        server = eventlet.listen(self._address)
        eventlet.spawn(self._broadcast)
        while True:
            sock, address = server.accept()
            print "Accepted connection from {}".format(address)
            self._clients[address] = sock
            self._pool.spawn_n(self._handle_client, sock, address)
    
    def _handle_client(self, client, address):
        buffer = LineBuffer(self._receive, address)
        while True:
            buffer.receive(client.recv(4096))
    
    def _receive(self, message, address):
        self._handle_message(message, address)
    
    def _handle_message(self, message, address):
        try:
            structure = json.loads(message)
        except ValueError:
            structure = {"id": str(uuid.uuid4()),
                         "message": message.rstrip()}
        
        payload = structure['message']
        if payload.startswith("connect "):
            address = payload[8:].split(':')
            self._connect((address[0], int(address[1])))
        elif payload == "source":
            self._clients[address].sendall(open(__file__).read())
        else:
            structure['address'] = address
            self._send(json.dumps(structure), address)
    
    def _send(self, message, address):
        self._queue.put((message, address))
    
    def _broadcast(self):
        while True:
            message, address = self._queue.get()
            try:
                payload = json.loads(message)
            except ValueError:
                print "Unable to send message: '{!r}'".format(message)
                continue
            
            if payload['id'] not in self._messages:
                self._messages = [payload['id']] + self._messages[:256]
                for client_address, client in self._clients.items():
                    if client_address != address:
                        client.sendall(message + "\n")
    
    def _connect(self, address):
        sock = eventlet.connect(address)
        self._clients[address] = sock
        self._pool.spawn_n(self._handle_client, sock, address)
        print "Connected to {!r}".format(address)

# 需要导入模块: from eventlet.queue import Queue [as 别名]
# 或者: from eventlet.queue.Queue import get [as 别名]

#.........这里部分代码省略.........
                if i.wd_routes:
                    for j in i.wd_routes:
                        entry = route_entry.BGPEntry(j.network, j.length, _4or6)
                        entry.announcer = netaddr.IPAddress(self.address[0])
                        withdraw_entries.append(entry)
        self.__add_route(advert_entries, attributes)
        self.__remove_route(withdraw_entries)

    def __add_route(self, advert_entries, attributes):
        # XXX acquire route table lock?
        # XXX remove duplicate
        for entry in advert_entries:
            entry.attributes = attributes
            Server.route_table.append(entry)

    def __remove_route(self, withdraw_entries):
        # XXX acquire route table lock?
        for i in withdraw_entries:
            for j in Server.route_table:
                if i == j:
                    Server.route_table.remove(j)

    def _handle_notification(self, msg):
        LOG.error('BGP error code %s, error sub code %s',
                  msg.err_code, msg.err_subcode)

    def _handle_keepalive(self, msg):
        self.peer_last_keepalive_timestamp = time.time()

    @_deactivate
    def _send_loop(self):
        try:
            while self.is_active:
                buf = self.send_q.get()
                self.socket.sendall(buf)
        finally:
            self.send_q = None

    def send(self, buf):
        if self.send_q:
            self.send_q.put(buf)

    def serve(self):
        send_thr = hub.spawn(self._send_loop)

        try:
            self._recv_loop()
        finally:
            hub.kill(send_thr)
            hub.joinall([send_thr])

    #
    #  Utility methods for convenience
    #  

    def send_open_msg(self):
        open_reply = BGP4.bgp4_open(version=4, my_as=Server.local_as,
                                    hold_time=self.hold_time,
                                    bgp_identifier=Server.local_ipv4,
                                    data=Server.capabilities)
        bgp4_reply = BGP4.bgp4(type_=BGP4.BGP4_OPEN, data=open_reply)
        self.serialize_and_send(bgp4_reply)

    def send_keepalive_msg(self):
        keepalive = BGP4.bgp4(type_=BGP4.BGP4_KEEPALIVE, data=None)
        self.serialize_and_send(keepalive)

# 需要导入模块: from eventlet.queue import Queue [as 别名]
# 或者: from eventlet.queue.Queue import get [as 别名]
class StatsdLog(object):

    """Simple server to monitor a syslog udp stream for statsd events"""

    def __init__(self, conf):
        TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))
        self.conf = conf
        self.logger = logging.getLogger('statsdlogd')
        self.logger.setLevel(logging.INFO)
        self.syslog = SysLogHandler(address='/dev/log')
        self.formatter = logging.Formatter('%(name)s: %(message)s')
        self.syslog.setFormatter(self.formatter)
        self.logger.addHandler(self.syslog)
        self.debug = conf.get('debug', 'false').lower() in TRUE_VALUES
        self.statsd_host = conf.get('statsd_host', '127.0.0.1')
        self.statsd_port = int(conf.get('statsd_port', '8125'))
        self.listen_addr = conf.get('listen_addr', '127.0.0.1')
        self.listen_port = int(conf.get('listen_port', 8126))
        self.report_internal_stats = conf.get('report_internal_stats',
                                              'true').lower() in TRUE_VALUES
        self.int_stats_interval = int(conf.get('internal_stats_interval', 5))
        self.buff = int(conf.get('buffer_size', 8192))
        self.max_q_size = int(conf.get('max_line_backlog', 512))
        self.statsd_sample_rate = float(conf.get('statsd_sample_rate', '.5'))
        self.counter = 0
        self.skip_counter = 0
        self.hits = 0
        self.q = Queue(maxsize=self.max_q_size)
        # key: regex
        self.patterns_file = conf.get('patterns_file',
                                      '/etc/statsdlog/patterns.json')
        self.json_patterns = conf.get('json_pattern_file',
                                      'true').lower() in TRUE_VALUES
        try:
            self.patterns = self.load_patterns()
        except Exception as err:
            self.logger.exception(err)
            print "Encountered exception at startup: %s" % err
            sys.exit(1)
        self.statsd_addr = (self.statsd_host, self.statsd_port)
        self.comp_patterns = {}
        for item in self.patterns:
            self.comp_patterns[item] = re.compile(self.patterns[item])

    def load_patterns(self):
        if self.json_patterns:
            self.logger.info("Using json based patterns file: %s" %
                             self.patterns_file)
            with open(self.patterns_file) as pfile:
                return json.loads(pfile.read())
        else:
            self.logger.info("Using plain text patterns file: %s" %
                             self.patterns_file)
        patterns = {}
        with open(self.patterns_file) as f:
            for line in f:
                if line:
                    pattern = [x.strip() for x in line.split("=", 1)]
                else:
                    pattern = None
                if len(pattern) != 2:
                    # skip this line
                    self.logger.error(
                        "Skipping pattern. Unable to parse: %s" % line)
                else:
                    if pattern[0] and pattern[1]:
                        patterns[pattern[0]] = pattern[1]
                    else:
                        self.logger.error(
                            "Skipping pattern. Unable to parse: %s" % line)
        return patterns

    def check_line(self, line):
        """
        Check if a line matches our search patterns.

        :param line: The string to check
        :returns: List of regex entries that matched (or empty list if none)
        """
        matches = []
        for entry in self.comp_patterns:
            if self.comp_patterns[entry].match(line):
                matches.append(entry)
        return matches

    def internal_stats(self):
        """
        Periodically send our own stats to statsd.
        """
        lastcount = 0
        lasthit = 0
        while True:
            eventlet.sleep(self.int_stats_interval)
            self.send_event("statsdlog.lines:%s|c" %
                            (self.counter - lastcount))
            lastcount = self.counter
            self.send_event("statsdlog.hits:%s|c" % (self.hits - lasthit))
            lasthit = self.hits

    def stats_print(self):
#.........这里部分代码省略.........

# 需要导入模块: from eventlet.queue import Queue [as 别名]
# 或者: from eventlet.queue.Queue import get [as 别名]
class SockWrapper(object):
    """ base class for SFK and Client(Rengine) sockets wrappers """

    NATIVE_PACKET = ScpPacket # placeholder, change it in successors
    
    def __init__(self):
        self.queue_send = Queue()
        self.queue_recv = Queue()
        self.appid = None

        # interface
        self.sock = None

    # interface for packet dispatchers - dockers

    def put_packet(self, packet):
        self.queue_send.put(packet)

    def get_packet(self):
        return self.queue_recv.get()


    # sender and recver started as greenthreads

    def sender(self, callback=lambda: None): 
        """get packet from sending queue, send it via sock. 
           By convention, packet type checking performed 
           before putting in queue
        """
        try:
            while True:
                packet = self.queue_send.get()
                data = packet.assemble()
                self.sock.sendall(data)
                # TODO if DEBUG
                try:
                    if packet.get_msg_type() == 'pong':
                        LOGGER.debug('pong sent %s' % self)
                except AttributeError:
                    pass

        except Exception:
            LOGGER.error(str(self) + " sender error")
        eventlet.spawn_n(callback)


    def recver(self, callback=lambda: None):
        """ recieve packets from sock, 
            check packet's type, 
            put packet to recv queue
        """
        f = self.sock.makefile()
        try:
            while True:
                try:
                    packet_class = determine_packet_type(f)
                except Disconnection as e:
                    raise Disconnection

                if packet_class == self.NATIVE_PACKET:
                    packet = packet_class()
                    packet.read_fields(f)  
                    self.queue_recv.put(packet)
                else:
                    LOGGER.error(
                        "{0} recver: unexpected magic".format(str(self)))
                    raise UnexpectedProtocol
                
        except Disconnection as e:
            LOGGER.info("Disconnection: {0}".format(str(self)))
        except Exception as e:
            LOGGER.error("recver error: {0} {1}".format(str(self), str(e)))
        LOGGER.info(str(self) + " recver terminate")
        eventlet.spawn_n(callback)


    def close_socket(self):
        try:
            self.sock.shutdown(socket.SHUT_RDWR)
            self.sock.close()
            LOGGER.info("{0} sockwrapper close socket".format(str(self)))
        except Exception as e:
            LOGGER.error("Fails socket close: %s" % str(e))