Python queue.LightQueue类代码示例

class ProducerPool(object):
    """Usage::

        >>> app = Celery(broker='amqp://')
        >>> ProducerPool(app)

    """
    Receipt = Receipt

    def __init__(self, app, size=20):
        self.app = app
        self.size = size
        self.inqueue = LightQueue()
        self._running = None
        self._producers = None

    def apply_async(self, task, args, kwargs, callback=None, **options):
        if self._running is None:
            self._running = spawn_n(self._run)
        receipt = self.Receipt(callback)
        self.inqueue.put((task, args, kwargs, options, receipt))
        return receipt

    def _run(self):
        self._producers = [spawn_n(self._producer) for _ in range(self.size)]

    def _producer(self):
        inqueue = self.inqueue

        with self.app.producer_or_acquire() as producer:
            while 1:
                task, args, kwargs, options, receipt = inqueue.get()
                result = task.apply_async(
                    args, kwargs, producer=producer, **options)
                receipt.finished(result)

class ProducerPool(object):
    Receipt = Receipt

    def __init__(self, size=20):
        self.size = size
        self.inqueue = LightQueue()
        self._running = None
        self._producers = None

    def apply_async(self, task, args, kwargs, callback=None, **options):
        if self._running is None:
            self._running = spawn_n(self._run)
        receipt = self.Receipt(callback)
        self.inqueue.put((task, args, kwargs, options, receipt))
        return receipt

    def _run(self):
        self._producers = [spawn_n(self._producer)
                                for _ in xrange(self.size)]

    def _producer(self):
        connection = current_app.broker_connection()
        publisher = current_app.amqp.TaskPublisher(connection)
        inqueue = self.inqueue

        while 1:
            task, args, kwargs, options, receipt = inqueue.get()
            result = task.apply_async(args, kwargs,
                                      publisher=publisher,
                                      **options)
            receipt.finished(result)

    def __init__(self, name, socket=None):
        """Takes a string and maybe a socket.

        If given a socket, we will try to play nice with its loop.

        """
        LightQueue.__init__(self)
        self._socket = socket
        self._name = name

 def get(self, timeout=10):  # pylint: disable=missing-docstring
     name = getter.__name__
     signal = '{uuid}{sep}{event}'.format(
         uuid=self._uuid,  # pylint: disable=protected-access
         sep=SEPARATOR,
         event=name
     )
     event = LightQueue(1)
     if flask.has_request_context():
         emit(signal, callback=lambda x: event.put(unpack(x)))
     else:
         sio = flask.current_app.extensions['socketio']
         sio.emit(signal, callback=lambda x: event.put(unpack(x)))
     data = event.get(timeout=timeout)
     return getter(self, data)

 def __init__(self, connection_class=Connection, max_connections=None,
              **connection_kwargs):
     self.pid = os.getpid()
     self.connection_class = connection_class
     self.connection_kwargs = connection_kwargs
     self.max_connections = max_connections or 2 ** 31
     self._created_connections = 0
     self._available_connections = LightQueue()
     self._in_use_connections = set()

class LocalMailbox(Mailbox):
    __slots__ = ['_queue']

    def __init__(self):
        self._queue = LightQueue()

    def put(self, message):
        self._queue.put(message)

    def get(self):
        return self._queue.get(block=True)

    def encode(self):
        raise NotImplementedError

    @staticmethod
    def decode(params):
        raise NotImplementedError

def fail_fast_imap(pool, call, items):
    """ Run a function against each item in a given list, yielding each
    function result in turn, where the function call is handled in a
    :class:`~eventlet.greenthread.GreenThread` spawned by the provided pool.

    If any function raises an exception, all other ongoing threads are killed,
    and the exception is raised to the caller.

    This function is similar to :meth:`~eventlet.greenpool.GreenPool.imap`.

    :param pool: Pool to spawn function threads from
    :type pool: eventlet.greenpool.GreenPool
    :param call: Function call to make, expecting to receive an item from the
        given list
    """
    result_queue = LightQueue(maxsize=len(items))
    spawned_threads = set()

    def handle_result(finished_thread):
        try:
            thread_result = finished_thread.wait()
            spawned_threads.remove(finished_thread)
            result_queue.put((thread_result, None))
        except Exception:
            spawned_threads.remove(finished_thread)
            result_queue.put((None, sys.exc_info()))

    for item in items:
        gt = pool.spawn(call, item)
        spawned_threads.add(gt)
        gt.link(handle_result)

    while spawned_threads:
        result, exc_info = result_queue.get()
        if exc_info is not None:
            # Kill all other ongoing threads
            for ongoing_thread in spawned_threads:
                ongoing_thread.kill()
            # simply raising here (even raising a full exc_info) isn't
            # sufficient to preserve the original stack trace.
            # greenlet.throw() achieves this.
            eventlet.getcurrent().throw(*exc_info)
        yield result

 def __init__(self, interval=None, queue=None, set_as_current=True):
     self.set_as_current = set_as_current
     if self.set_as_current:
         set_current(self)
     self._orig_queue_arg = queue
     self.interval = interval or self.interval
     self.queue = LightQueue() if queue is None else queue
     self._pause_mutex = Lock()
     self._last_update = None
     gThread.__init__(self)
     Status.__init__(self)

def fail_fast_imap(pool, call, items):
    """ Run a function against each item in a given list, yielding each
    function result in turn, where the function call is handled in a
    :class:`~eventlet.greenthread.GreenThread` spawned by the provided pool.

    If any function raises an exception, all other ongoing threads are killed,
    and the exception is raised to the caller.

    This function is similar to :meth:`~eventlet.greenpool.GreenPool.imap`.

    :param pool: Pool to spawn function threads from
    :type pool: eventlet.greenpool.GreenPool
    :param call: Function call to make, expecting to receive an item from the
        given list
    """
    result_queue = LightQueue(maxsize=len(items))
    spawned_threads = set()

    def handle_result(finished_thread):
        try:
            thread_result = finished_thread.wait()
            spawned_threads.remove(finished_thread)
            result_queue.put((thread_result, None))
        except Exception as e:
            spawned_threads.remove(finished_thread)
            result_queue.put((None, e))

    for item in items:
        gt = pool.spawn(call, item)
        spawned_threads.add(gt)
        gt.link(handle_result)

    while spawned_threads:
        result, raised_exc = result_queue.get()
        if raised_exc is not None:
            # Kill all other ongoing threads
            for ongoing_thread in spawned_threads:
                ongoing_thread.kill()
            raise raised_exc
        yield result

    def __getitem__(self, key):
        """Load the value stored with the key.

        Parameters
        ----------
        key : str
            The key to lookup the value stored.

        Returns
        -------
        object
            The value if the key exists in the cache, otherwise None.

        """
        validate(key)
        signal = 'cache_load'
        event = LightQueue(1)
        if flask.has_request_context():
            emit(signal, {'data': pack(key)}, callback=event.put)
        else:
            sio = flask.current_app.extensions['socketio']
            sio.emit(signal, {'data': pack(key)}, callback=event.put)
        return msgpack.unpackb(bytes(event.get(timeout=10)), encoding='utf8')

class EventletConnectionPool(ConnectionPool):
    def __init__(self, connection_class=Connection, max_connections=None,
                 **connection_kwargs):
        self.pid = os.getpid()
        self.connection_class = connection_class
        self.connection_kwargs = connection_kwargs
        self.max_connections = max_connections or 2 ** 31
        self._created_connections = 0
        self._available_connections = LightQueue()
        self._in_use_connections = set()
    def get_connection(self, command_name, *keys, **options):
        "Get a connection from the pool"
        try:
            connection = self._available_connections.get_nowait()
        except Empty:
            if self._created_connections < self.max_connections:
                connection = self.make_connection()
            else:
                try:
                    connection = self._available_connections.get()
                except Empty:
                    raise ConnectionError("Couldn't find a free connection")
        self._in_use_connections.add(connection)
        return connection
    def release(self, connection):
        "Releases the connection back to the pool"
        self._checkpid()
        if connection.pid == self.pid:
            self._in_use_connections.remove(connection)
            self._available_connections.put_nowait(connection)
    def disconnect(self):
        "Disconnects all connections in the pool"
        while True:
            try:
                self._available_connections.get_nowait().disconnect()
            except Empty:
                break
        for connection in self._in_use_connections:
            connection.disconnect()

 def __init__(self, maxsize, transfer_size):
     LightQueue.__init__(self, maxsize)
     self.transfer_size = transfer_size
     self.transferred = 0

 def __init__(self, app, size=20):
     self.app = app
     self.size = size
     self.inqueue = LightQueue()
     self._running = None
     self._producers = None

class Supervisor(gThread, Status):
    """The supervisor wakes up at intervals to monitor changes in the model.
    It can also be requested to perform specific operations, and these
    operations can be either async or sync.

    :keyword interval:  This is the interval (in seconds as an int/float),
       between verifying all the registered instances.
    :keyword queue: Custom :class:`~Queue.Queue` instance used to send
        and receive commands.

    It is responsible for:

        * Stopping removed instances.
        * Starting new instances.
        * Restarting unresponsive/killed instances.
        * Making sure the instances consumes from the queues
          specified in the model, sending ``add_consumer``/-
          ``cancel_consumer`` broadcast commands to the instances as it
          finds inconsistencies.
        * Making sure the max/min concurrency setting is as specified in the
          model,  sending ``autoscale`` broadcast commands to the noes
          as it finds inconsistencies.

    The supervisor is resilient to intermittent connection failures,
    and will auto-retry any operation that is dependent on a broker.

    Since workers cannot respond to broadcast commands while the
    broker is off-line, the supervisor will not restart affected
    instances until the instance has had a chance to reconnect (decided
    by the :attr:`wait_after_broker_revived` attribute).

    """
    #: Limit instance restarts to 1/m, so out of control
    #: instances will be disabled
    restart_max_rate = '1/m'

    #: Default interval_max for ensure_connection is 30 secs.
    wait_after_broker_revived = 35.0

    #: Connection errors pauses the supervisor, so events does not accumulate.
    paused = False

    #: Default interval (time in seconds as a float to reschedule).
    interval = 60.0

    def __init__(self, interval=None, queue=None, set_as_current=True):
        self.set_as_current = set_as_current
        if self.set_as_current:
            set_current(self)
        self._orig_queue_arg = queue
        self.interval = interval or self.interval
        self.queue = LightQueue() if queue is None else queue
        self._pause_mutex = Lock()
        self._last_update = None
        gThread.__init__(self)
        Status.__init__(self)

    def __copy__(self):
        return self.__class__(self.interval, self._orig_queue_arg)

    def pause(self):
        """Pause all timers."""
        self.respond_to_ping()
        with self._pause_mutex:
            if not self.paused:
                self.debug('pausing')
                self.paused = True

    def resume(self):
        """Resume all timers."""
        with self._pause_mutex:
            if self.paused:
                self.debug('resuming')
                self.paused = False

    def verify(self, instances, ratelimit=False):
        """Verify the consistency of one or more instances.

        :param instances: List of instances to verify.

        This operation is asynchronous, and returns a :class:`Greenlet`
        instance that can be used to wait for the operation to complete.

        """
        return self._request(instances, self._do_verify_instance,
                            {'ratelimit': ratelimit})

    def restart(self, instances):
        """Restart one or more instances.

        :param instances: List of instances to restart.

        This operation is asynchronous, and returns a :class:`Greenlet`
        instance that can be used to wait for the operation to complete.

        """
        return self._request(instances, self._do_restart_instance)

    def shutdown(self, instances):
        """Shutdown one or more instances.
#.........这里部分代码省略.........

 def __init__(self):
     self._queue = LightQueue()