Python LRUCache.update方法代码示例

# 需要导入模块: from celery.datastructures import LRUCache [as 别名]
# 或者: from celery.datastructures.LRUCache import update [as 别名]
    def assertSafeIter(self, method, interval=0.01, size=10000):
        from threading import Thread, Event
        from time import sleep
        x = LRUCache(size)
        x.update(zip(xrange(size), xrange(size)))

        class Burglar(Thread):

            def __init__(self, cache):
                self.cache = cache
                self._is_shutdown = Event()
                self._is_stopped = Event()
                Thread.__init__(self)

            def run(self):
                while not self._is_shutdown.isSet():
                    try:
                        self.cache.data.popitem(last=False)
                    except KeyError:
                        break
                self._is_stopped.set()

            def stop(self):
                self._is_shutdown.set()
                self._is_stopped.wait()
                self.join(THREAD_TIMEOUT_MAX)

        burglar = Burglar(x)
        burglar.start()
        try:
            for _ in getattr(x, method)():
                sleep(0.0001)
        finally:
            burglar.stop()

# 需要导入模块: from celery.datastructures import LRUCache [as 别名]
# 或者: from celery.datastructures.LRUCache import update [as 别名]
    def test_update_expires(self):
        limit = 100
        x = LRUCache(limit=limit)
        slots = list(xrange(limit * 2))
        for i in slots:
            x.update({i: i})

        self.assertListEqual(list(x.keys()), list(slots[limit:]))

# 需要导入模块: from celery.datastructures import LRUCache [as 别名]
# 或者: from celery.datastructures.LRUCache import update [as 别名]
 def test_items(self):
     c = LRUCache()
     c.update(a=1, b=2, c=3)
     self.assertTrue(c.items())

# 需要导入模块: from celery.datastructures import LRUCache [as 别名]
# 或者: from celery.datastructures.LRUCache import update [as 别名]
class State(object):
    """Records clusters state."""
    event_count = 0
    task_count = 0

    def __init__(self, callback=None,
            max_workers_in_memory=5000, max_tasks_in_memory=10000):
        self.workers = LRUCache(limit=max_workers_in_memory)
        self.tasks = LRUCache(limit=max_tasks_in_memory)
        self.event_callback = callback
        self.group_handlers = {'worker': self.worker_event,
                               'task': self.task_event}
        self._mutex = threading.Lock()

    def freeze_while(self, fun, *args, **kwargs):
        clear_after = kwargs.pop('clear_after', False)
        with self._mutex:
            try:
                return fun(*args, **kwargs)
            finally:
                if clear_after:
                    self._clear()

    def clear_tasks(self, ready=True):
        with self._mutex:
            return self._clear_tasks(ready)

    def _clear_tasks(self, ready=True):
        if ready:
            in_progress = dict((uuid, task) for uuid, task in self.itertasks()
                                if task.state not in states.READY_STATES)
            self.tasks.clear()
            self.tasks.update(in_progress)
        else:
            self.tasks.clear()

    def _clear(self, ready=True):
        self.workers.clear()
        self._clear_tasks(ready)
        self.event_count = 0
        self.task_count = 0

    def clear(self, ready=True):
        with self._mutex:
            return self._clear(ready)

    def get_or_create_worker(self, hostname, **kwargs):
        """Get or create worker by hostname."""
        try:
            worker = self.workers[hostname]
            worker.update(kwargs)
        except KeyError:
            worker = self.workers[hostname] = Worker(
                    hostname=hostname, **kwargs)
        return worker

    def get_or_create_task(self, uuid):
        """Get or create task by uuid."""
        try:
            return self.tasks[uuid]
        except KeyError:
            task = self.tasks[uuid] = Task(uuid=uuid)
            return task

    def worker_event(self, type, fields):
        """Process worker event."""
        hostname = fields.pop('hostname', None)
        if hostname:
            worker = self.get_or_create_worker(hostname)
            handler = getattr(worker, 'on_%s' % type, None)
            if handler:
                handler(**fields)

    def task_event(self, type, fields):
        """Process task event."""
        uuid = fields.pop('uuid')
        hostname = fields.pop('hostname')
        worker = self.get_or_create_worker(hostname)
        task = self.get_or_create_task(uuid)
        handler = getattr(task, 'on_%s' % type, None)
        if type == 'received':
            self.task_count += 1
        if handler:
            handler(**fields)
        else:
            task.on_unknown_event(type, **fields)
        task.worker = worker

    def event(self, event):
        with self._mutex:
            return self._dispatch_event(event)

    def _dispatch_event(self, event):
        self.event_count += 1
        event = kwdict(event)
        group, _, type = event.pop('type').partition('-')
        self.group_handlers[group](type, event)
        if self.event_callback:
            self.event_callback(self, event)

#.........这里部分代码省略.........

# 需要导入模块: from celery.datastructures import LRUCache [as 别名]
# 或者: from celery.datastructures.LRUCache import update [as 别名]
class State(object):
    """Records clusters state."""

    event_count = 0
    task_count = 0

    def __init__(self, callback=None, max_workers_in_memory=5000, max_tasks_in_memory=10000):
        self.max_workers_in_memory = max_workers_in_memory
        self.max_tasks_in_memory = max_tasks_in_memory
        self.workers = LRUCache(limit=self.max_workers_in_memory)
        self.tasks = LRUCache(limit=self.max_tasks_in_memory)
        self._taskheap = []
        self.event_callback = callback
        self.group_handlers = {"worker": self.worker_event, "task": self.task_event}
        self._mutex = threading.Lock()

    def freeze_while(self, fun, *args, **kwargs):
        clear_after = kwargs.pop("clear_after", False)
        with self._mutex:
            try:
                return fun(*args, **kwargs)
            finally:
                if clear_after:
                    self._clear()

    def clear_tasks(self, ready=True):
        with self._mutex:
            return self._clear_tasks(ready)

    def _clear_tasks(self, ready=True):
        if ready:
            in_progress = dict((uuid, task) for uuid, task in self.itertasks() if task.state not in states.READY_STATES)
            self.tasks.clear()
            self.tasks.update(in_progress)
        else:
            self.tasks.clear()
        self._taskheap[:] = []

    def _clear(self, ready=True):
        self.workers.clear()
        self._clear_tasks(ready)
        self.event_count = 0
        self.task_count = 0

    def clear(self, ready=True):
        with self._mutex:
            return self._clear(ready)

    def get_or_create_worker(self, hostname, **kwargs):
        """Get or create worker by hostname.

        Returns tuple of ``(worker, was_created)``.
        """
        try:
            worker = self.workers[hostname]
            worker.update(kwargs)
            return worker, False
        except KeyError:
            worker = self.workers[hostname] = Worker(hostname=hostname, **kwargs)
            return worker, True

    def get_or_create_task(self, uuid):
        """Get or create task by uuid."""
        try:
            return self.tasks[uuid], True
        except KeyError:
            task = self.tasks[uuid] = Task(uuid=uuid)
            return task, False

    def worker_event(self, type, fields):
        """Process worker event."""
        try:
            hostname = fields["hostname"]
        except KeyError:
            pass
        else:
            worker, created = self.get_or_create_worker(hostname)
            handler = getattr(worker, "on_" + type, None)
            if handler:
                handler(**fields)
            return worker, created

    def task_event(self, type, fields):
        """Process task event."""
        uuid = fields.pop("uuid")
        hostname = fields.pop("hostname")
        worker, _ = self.get_or_create_worker(hostname)
        task, created = self.get_or_create_task(uuid)
        task.worker = worker

        taskheap = self._taskheap
        timestamp = fields.get("timestamp") or 0
        clock = 0 if type == "sent" else fields.get("clock")
        heappush(taskheap, _lamportinfo(clock, timestamp, worker.id, task))
        curcount = len(self.tasks)
        if len(taskheap) > self.max_tasks_in_memory * 2:
            taskheap[:] = taskheap[curcount:]

        handler = getattr(task, "on_" + type, None)
        if type == "received":
#.........这里部分代码省略.........

# 需要导入模块: from celery.datastructures import LRUCache [as 别名]
# 或者: from celery.datastructures.LRUCache import update [as 别名]
 def test_items(self):
     c = LRUCache()
     c.update(a=1, b=2, c=3)
     self.assertTrue(list(items(c)))