Python LRUCache.update方法代码示例

# 需要导入模块: from celery.utils.functional import LRUCache [as 别名]
# 或者: from celery.utils.functional.LRUCache import update [as 别名]
    def assertSafeIter(self, method, interval=0.01, size=10000):
        from threading import Thread, Event
        from time import sleep
        x = LRUCache(size)
        x.update(zip(range(size), range(size)))

        class Burglar(Thread):

            def __init__(self, cache):
                self.cache = cache
                self.__is_shutdown = Event()
                self.__is_stopped = Event()
                Thread.__init__(self)

            def run(self):
                while not self.__is_shutdown.isSet():
                    try:
                        self.cache.data.popitem(last=False)
                    except KeyError:
                        break
                self.__is_stopped.set()

            def stop(self):
                self.__is_shutdown.set()
                self.__is_stopped.wait()
                self.join(THREAD_TIMEOUT_MAX)

        burglar = Burglar(x)
        burglar.start()
        try:
            for _ in getattr(x, method)():
                sleep(0.0001)
        finally:
            burglar.stop()

# 需要导入模块: from celery.utils.functional import LRUCache [as 别名]
# 或者: from celery.utils.functional.LRUCache import update [as 别名]
    def test_update_expires(self):
        limit = 100
        x = LRUCache(limit=limit)
        slots = list(range(limit * 2))
        for i in slots:
            x.update({i: i})

        self.assertListEqual(list(x.keys()), list(slots[limit:]))

# 需要导入模块: from celery.utils.functional import LRUCache [as 别名]
# 或者: from celery.utils.functional.LRUCache import update [as 别名]
 def test_is_pickleable(self):
     x = LRUCache(limit=10)
     x.update(luke=1, leia=2)
     y = pickle.loads(pickle.dumps(x))
     self.assertEqual(y.limit, y.limit)
     self.assertEqual(y, x)

# 需要导入模块: from celery.utils.functional import LRUCache [as 别名]
# 或者: from celery.utils.functional.LRUCache import update [as 别名]
 def test_items(self):
     c = LRUCache()
     c.update(a=1, b=2, c=3)
     self.assertTrue(list(items(c)))

# 需要导入模块: from celery.utils.functional import LRUCache [as 别名]
# 或者: from celery.utils.functional.LRUCache import update [as 别名]
class State(object):
    """Records clusters state."""
    event_count = 0
    task_count = 0

    def __init__(self, callback=None,
                 max_workers_in_memory=5000, max_tasks_in_memory=10000):
        self.max_workers_in_memory = max_workers_in_memory
        self.max_tasks_in_memory = max_tasks_in_memory
        self.workers = LRUCache(limit=self.max_workers_in_memory)
        self.tasks = LRUCache(limit=self.max_tasks_in_memory)
        self._taskheap = []
        self.event_callback = callback
        self._mutex = threading.Lock()

    def freeze_while(self, fun, *args, **kwargs):
        clear_after = kwargs.pop('clear_after', False)
        with self._mutex:
            try:
                return fun(*args, **kwargs)
            finally:
                if clear_after:
                    self._clear()

    def clear_tasks(self, ready=True):
        with self._mutex:
            return self._clear_tasks(ready)

    def _clear_tasks(self, ready=True):
        if ready:
            in_progress = dict(
                (uuid, task) for uuid, task in self.itertasks()
                if task.state not in states.READY_STATES)
            self.tasks.clear()
            self.tasks.update(in_progress)
        else:
            self.tasks.clear()
        self._taskheap[:] = []

    def _clear(self, ready=True):
        self.workers.clear()
        self._clear_tasks(ready)
        self.event_count = 0
        self.task_count = 0

    def clear(self, ready=True):
        with self._mutex:
            return self._clear(ready)

    def get_or_create_worker(self, hostname, **kwargs):
        """Get or create worker by hostname.

        Returns tuple of ``(worker, was_created)``.
        """
        try:
            worker = self.workers[hostname]
            worker.update(kwargs)
            return worker, False
        except KeyError:
            worker = self.workers[hostname] = Worker(
                hostname=hostname, **kwargs)
            return worker, True

    def get_or_create_task(self, uuid):
        """Get or create task by uuid."""
        try:
            return self.tasks[uuid], True
        except KeyError:
            task = self.tasks[uuid] = Task(uuid=uuid)
            return task, False

    def worker_event(self, type, fields):
        """Process worker event."""
        try:
            hostname = fields['hostname']
        except KeyError:
            pass
        else:
            worker, created = self.get_or_create_worker(hostname)
            handler = getattr(worker, 'on_' + type, None)
            if handler:
                handler(**fields)
            return worker, created

    def task_event(self, type, fields):
        """Process task event."""
        uuid = fields['uuid']
        hostname = fields['hostname']
        worker, _ = self.get_or_create_worker(hostname)
        task, created = self.get_or_create_task(uuid)
        task.worker = worker
        maxtasks = self.max_tasks_in_memory * 2

        taskheap = self._taskheap
        timestamp = fields.get('timestamp') or 0
        clock = 0 if type == 'sent' else fields.get('clock')
        heappush(taskheap, _lamportinfo(clock, timestamp, worker.id, task))
        if len(taskheap) > maxtasks:
            heappop(taskheap)

#.........这里部分代码省略.........

# 需要导入模块: from celery.utils.functional import LRUCache [as 别名]
# 或者: from celery.utils.functional.LRUCache import update [as 别名]
 def test_update_larger_than_cache_size(self):
     x = LRUCache(2)
     x.update(dict((x, x) for x in range(100)))
     self.assertEqual(list(x.keys()), [98, 99])

# 需要导入模块: from celery.utils.functional import LRUCache [as 别名]
# 或者: from celery.utils.functional.LRUCache import update [as 别名]
class State(object):
    """Records clusters state."""

    Worker = Worker
    Task = Task
    event_count = 0
    task_count = 0
    heap_multiplier = 4

    def __init__(
        self,
        callback=None,
        workers=None,
        tasks=None,
        taskheap=None,
        max_workers_in_memory=5000,
        max_tasks_in_memory=10000,
        on_node_join=None,
        on_node_leave=None,
    ):
        self.event_callback = callback
        self.workers = LRUCache(max_workers_in_memory) if workers is None else workers
        self.tasks = LRUCache(max_tasks_in_memory) if tasks is None else tasks
        self._taskheap = [] if taskheap is None else taskheap
        self.max_workers_in_memory = max_workers_in_memory
        self.max_tasks_in_memory = max_tasks_in_memory
        self.on_node_join = on_node_join
        self.on_node_leave = on_node_leave
        self._mutex = threading.Lock()
        self.handlers = {}
        self._seen_types = set()
        self.rebuild_taskheap()

    @cached_property
    def _event(self):
        return self._create_dispatcher()

    def freeze_while(self, fun, *args, **kwargs):
        clear_after = kwargs.pop("clear_after", False)
        with self._mutex:
            try:
                return fun(*args, **kwargs)
            finally:
                if clear_after:
                    self._clear()

    def clear_tasks(self, ready=True):
        with self._mutex:
            return self._clear_tasks(ready)

    def _clear_tasks(self, ready=True):
        if ready:
            in_progress = {uuid: task for uuid, task in self.itertasks() if task.state not in states.READY_STATES}
            self.tasks.clear()
            self.tasks.update(in_progress)
        else:
            self.tasks.clear()
        self._taskheap[:] = []

    def _clear(self, ready=True):
        self.workers.clear()
        self._clear_tasks(ready)
        self.event_count = 0
        self.task_count = 0

    def clear(self, ready=True):
        with self._mutex:
            return self._clear(ready)

    def get_or_create_worker(self, hostname, **kwargs):
        """Get or create worker by hostname.

        Return tuple of ``(worker, was_created)``.
        """
        try:
            worker = self.workers[hostname]
            if kwargs:
                worker.update(kwargs)
            return worker, False
        except KeyError:
            worker = self.workers[hostname] = self.Worker(hostname, **kwargs)
            return worker, True

    def get_or_create_task(self, uuid):
        """Get or create task by uuid."""
        try:
            return self.tasks[uuid], False
        except KeyError:
            task = self.tasks[uuid] = self.Task(uuid)
            return task, True

    def event(self, event):
        with self._mutex:
            return self._event(event)

    def task_event(self, type_, fields):
        """Deprecated, use :meth:`event`."""
        return self._event(dict(fields, type="-".join(["task", type_])))[0]

    def worker_event(self, type_, fields):
#.........这里部分代码省略.........

# 需要导入模块: from celery.utils.functional import LRUCache [as 别名]
# 或者: from celery.utils.functional.LRUCache import update [as 别名]
class State(object):
    """Records clusters state."""

    Worker = Worker
    Task = Task
    event_count = 0
    task_count = 0

    def __init__(
        self,
        callback=None,
        workers=None,
        tasks=None,
        taskheap=None,
        max_workers_in_memory=5000,
        max_tasks_in_memory=10000,
    ):
        self.event_callback = callback
        self.workers = LRUCache(max_workers_in_memory) if workers is None else workers
        self.tasks = LRUCache(max_tasks_in_memory) if tasks is None else tasks
        self._taskheap = [] if taskheap is None else taskheap
        self.max_workers_in_memory = max_workers_in_memory
        self.max_tasks_in_memory = max_tasks_in_memory
        self._mutex = threading.Lock()
        self.handlers = {}
        self._seen_types = set()
        self.rebuild_taskheap()

    @cached_property
    def _event(self):
        return self._create_dispatcher()

    def freeze_while(self, fun, *args, **kwargs):
        clear_after = kwargs.pop("clear_after", False)
        with self._mutex:
            try:
                return fun(*args, **kwargs)
            finally:
                if clear_after:
                    self._clear()

    def clear_tasks(self, ready=True):
        with self._mutex:
            return self._clear_tasks(ready)

    def _clear_tasks(self, ready=True):
        if ready:
            in_progress = dict((uuid, task) for uuid, task in self.itertasks() if task.state not in states.READY_STATES)
            self.tasks.clear()
            self.tasks.update(in_progress)
        else:
            self.tasks.clear()
        self._taskheap[:] = []

    def _clear(self, ready=True):
        self.workers.clear()
        self._clear_tasks(ready)
        self.event_count = 0
        self.task_count = 0

    def clear(self, ready=True):
        with self._mutex:
            return self._clear(ready)

    def get_or_create_worker(self, hostname, **kwargs):
        """Get or create worker by hostname.

        Return tuple of ``(worker, was_created)``.
        """
        try:
            worker = self.workers[hostname]
            if kwargs:
                worker.update(kwargs)
            return worker, False
        except KeyError:
            worker = self.workers[hostname] = self.Worker(hostname, **kwargs)
            return worker, True

    def get_or_create_task(self, uuid):
        """Get or create task by uuid."""
        try:
            return self.tasks[uuid], False
        except KeyError:
            task = self.tasks[uuid] = self.Task(uuid)
            return task, True

    def event(self, event):
        with self._mutex:
            return self._event(event)

    def task_event(self, type_, fields):
        """Deprecated, use :meth:`event`."""
        return self._event(dict(fields, type="-".join(["task", type_])))

    def worker_event(self, type_, fields):
        """Deprecated, use :meth:`event`."""
        return self._event(dict(fields, type="-".join(["worker", type_])))

    def _create_dispatcher(self):
        get_handler = self.handlers.__getitem__
#.........这里部分代码省略.........