Python RefHelper.discard_all方法代码示例

# 需要导入模块: from calico.felix.refcount import RefHelper [as 别名]
# 或者: from calico.felix.refcount.RefHelper import discard_all [as 别名]
class TestRefHelper(TestReferenceManager):
    def setUp(self):
        super(TestRefHelper, self).setUp()
        self._rh = RefHelper(self._rm,
                             self._rm,
                             self._rm.ready_callback)

    def test_no_refs(self):
        # With no references, we're ready but haven't been notified
        self.assertFalse(self._rm._ready_called)
        self.assertTrue(self._rh.ready)

        # Discarding non-existent references is allowed
        self._rh.discard_ref("foo")

    def test_acquire_discard_1(self):
        # Acquire a reference to 'foo' - it won't be ready immediately
        self._rh.acquire_ref("foo")
        self.assertFalse(self._rm._ready_called)
        self.assertFalse(self._rh.ready)

        # Spin the actor framework - we become ready
        _, obj = self.call_via_cb(self._rm.get_and_incref, "bar", async=True)
        self.assertTrue(self._rm._ready_called)
        self.assertTrue(self._rh.ready)
        self.assertEqual(next(self._rh.iteritems())[0], "foo")

        # Acquiring an already-acquired reference is idempotent
        self._rh.acquire_ref("foo")
        self.assertTrue(self._rh.ready)

        # Discard the reference
        self._rh.discard_ref("foo")
        _, obj = self.call_via_cb(self._rm.get_and_incref, "baz", async=True)
        self.assertTrue(self._rh.ready)

    def test_sync_acquire_discard(self):
        # Acquire a reference and discard it before it's become ready
        self._rh.acquire_ref("foo")
        self.assertFalse(self._rh.ready)

        self._rh.discard_ref("foo")
        self.assertTrue(self._rh.ready)

        # Spin the actor framework
        _, obj = self.call_via_cb(self._rm.get_and_incref, "bar", async=True)

    def test_acquire_discard_2(self):
        # Acquire two references
        self._rh.acquire_ref("foo")
        _, obj = self.call_via_cb(self._rm.get_and_incref, "bar", async=True)
        self._rh.acquire_ref("baz")
        self.assertFalse(self._rh.ready)
        _, obj = self.call_via_cb(self._rm.get_and_incref, "bar2", async=True)
        acq_ids = list(key for key, value in self._rh.iteritems())
        self.assertItemsEqual(acq_ids, ["foo", "baz"])
        self.assertTrue(self._rh.ready)

        # Discard them all!
        self._rh.discard_all()

# 需要导入模块: from calico.felix.refcount import RefHelper [as 别名]
# 或者: from calico.felix.refcount.RefHelper import discard_all [as 别名]

#.........这里部分代码省略.........
            # that iptables or the device is now out of sync.
            _log.debug("Endpoint update pending: %s", self._pending_endpoint)
            self._apply_endpoint_update()

        if not self._iptables_in_sync:
            # Try to update iptables, if successful, will set the
            # _iptables_in_sync flag.
            _log.debug("iptables is out-of-sync, trying to update it")
            if self._admin_up:
                _log.info("%s is 'active', (re)programming chains.", self)
                self._update_chains()
            elif self._chains_programmed:
                # No longer active but our chains are still in place.  Remove
                # them.
                _log.info("%s is not 'active', removing chains.", self)
                self._remove_chains()

        if not self._device_in_sync and self._iface_name:
            # Try to update the device configuration.  If successful, will set
            # the _device_in_sync flag.
            if self._admin_up:
                # Endpoint is supposed to be live, try to configure it.
                _log.debug("Device is out-of-sync, trying to configure it")
                self._configure_interface()
            else:
                # We've been deleted, de-configure the interface.
                _log.debug("Device is out-of-sync, trying to de-configure it")
                self._deconfigure_interface()

        if self._unreferenced:
            # Endpoint is being removed, clean up...
            _log.debug("Cleaning up after endpoint unreferenced")
            self.dispatch_chains.on_endpoint_removed(self._iface_name, async=True)
            self.rules_ref_helper.discard_all()
            self._notify_cleanup_complete()
            self._cleaned_up = True
        elif not self._added_to_dispatch_chains and self._iface_name:
            # This must be the first batch, add ourself to the dispatch chains.
            _log.debug("Adding endpoint to dispatch chain")
            self.dispatch_chains.on_endpoint_added(self._iface_name, async=True)
            self._added_to_dispatch_chains = True

        # If changed, report our status back to the datastore.
        self._maybe_update_status()

    def _maybe_update_status(self):
        if not self.config.REPORT_ENDPOINT_STATUS:
            _log.debug("Status reporting disabled. Not reporting status.")
            return

        if not self._device_is_up:
            # Check this first because we won't try to sync the device if it's
            # oper down.
            reason = "Interface is oper-down"
            status = ENDPOINT_STATUS_DOWN
        elif not self.endpoint:
            reason = "No endpoint data"
            status = ENDPOINT_STATUS_DOWN
        elif not self._iptables_in_sync:
            # Definitely an error, the iptables command failed.
            reason = "Failed to update iptables"
            status = ENDPOINT_STATUS_ERROR
        elif not self._device_in_sync:
            reason = "Failed to update device config"
            status = ENDPOINT_STATUS_ERROR
        elif not self._admin_up:

# 需要导入模块: from calico.felix.refcount import RefHelper [as 别名]
# 或者: from calico.felix.refcount.RefHelper import discard_all [as 别名]

#.........这里部分代码省略.........
                "Unexpected update to %s (%s) after being unreferenced" %
                (self, self.__dict__)
            )

        if self._endpoint_update_pending:
            # Copy the pending update into our data structures.  May work out
            # that iptables or the device is now out of sync.
            _log.debug("Endpoint update pending: %s", self._pending_endpoint)
            self._apply_endpoint_update()

        if not self._iptables_in_sync:
            # Try to update iptables, if successful, will set the
            # _iptables_in_sync flag.
            _log.debug("iptables is out-of-sync, trying to update it")
            self._maybe_update_iptables()

        if not self._device_in_sync and self._iface_name:
            # Try to update the device configuration.  If successful, will set
            # the _device_in_sync flag.
            if self.endpoint:
                # Endpoint is supposed to be live, try to configure it.
                _log.debug("Device is out-of-sync, trying to configure it")
                self._configure_interface()
            else:
                # We've been deleted, de-configure the interface.
                _log.debug("Device is out-of-sync, trying to de-configure it")
                self._deconfigure_interface()

        if self._unreferenced:
            # Endpoint is being removed, clean up...
            _log.debug("Cleaning up after endpoint unreferenced")
            self.dispatch_chains.on_endpoint_removed(self._iface_name,
                                                     async=True)
            self.rules_ref_helper.discard_all()
            self._notify_cleanup_complete()
            self._cleaned_up = True
        elif not self._added_to_dispatch_chains:
            # This must be the first batch, add ourself to the dispatch chains.
            _log.debug("Adding endpoint to dispatch chain")
            self.dispatch_chains.on_endpoint_added(self._iface_name,
                                                   async=True)
            self._added_to_dispatch_chains = True

    def _apply_endpoint_update(self):
        pending_endpoint = self._pending_endpoint
        if pending_endpoint == self.endpoint:
            _log.debug("Endpoint hasn't changed, nothing to do")
            return

        if pending_endpoint:
            # Update/create.
            if pending_endpoint['mac'] != self._mac:
                # Either we have not seen this MAC before, or it has changed.
                _log.debug("Endpoint MAC changed to %s",
                           pending_endpoint["mac"])
                self._mac = pending_endpoint['mac']
                self._mac_changed = True
                # MAC change requires refresh of iptables rules and ARP table.
                self._iptables_in_sync = False
                self._device_in_sync = False

            if self.endpoint is None:
                # This is the first time we have seen the endpoint, so extract
                # the interface name and endpoint ID.
                self._iface_name = pending_endpoint["name"]
                self._suffix = interface_to_suffix(self.config,

# 需要导入模块: from calico.felix.refcount import RefHelper [as 别名]
# 或者: from calico.felix.refcount.RefHelper import discard_all [as 别名]

#.........这里部分代码省略.........
            new_profile_ids = set(endpoint["profile_ids"])
        else:
            new_profile_ids = set()
        # Note: we don't actually need to wait for the activation to finish
        # due to the dependency management in the iptables layer.
        self.rules_ref_helper.replace_all(new_profile_ids)

        if endpoint != self.endpoint or force_reprogram:
            self._dirty = True

        # Store off the endpoint we were passed.
        self.endpoint = endpoint

        if endpoint:
            # Configure the network interface; may fail if not there yet (in
            # which case we'll just do it when the interface comes up).
            self._configure_interface(mac_changed)
        else:
            # Remove the network programming.
            self._deconfigure_interface()

        self._maybe_update(was_ready)
        _log.debug("%s finished processing update", self)

    @actor_message()
    def on_unreferenced(self):
        """
        Overrides RefCountedActor:on_unreferenced.
        """
        _log.info("%s now unreferenced, cleaning up", self)
        assert not self._ready, "Should be deleted before being unreffed."
        # Removing all profile refs should have been done already but be
        # defensive.
        self.rules_ref_helper.discard_all()
        self._notify_cleanup_complete()

    @actor_message()
    def on_interface_update(self):
        """
        Actor event to report that the interface is either up or changed.
        """
        _log.info("Endpoint %s received interface kick", self.combined_id)
        self._configure_interface()

    @property
    def _missing_deps(self):
        """
        Returns a list of missing dependencies.
        """
        missing_deps = []
        if not self.endpoint:
            missing_deps.append("endpoint")
        elif self.endpoint.get("state", "active") != "active":
            missing_deps.append("endpoint active")
        elif not self.endpoint.get("profile_ids"):
            missing_deps.append("profile")
        return missing_deps

    @property
    def _ready(self):
        """
        Returns whether this LocalEndpoint has any dependencies preventing it
        programming its rules.
        """
        return not self._missing_deps

# 需要导入模块: from calico.felix.refcount import RefHelper [as 别名]
# 或者: from calico.felix.refcount.RefHelper import discard_all [as 别名]
class ProfileRules(RefCountedActor):
    """
    Actor that owns the per-profile rules chains.
    """
    def __init__(self, iptables_generator, profile_id, ip_version,
                 iptables_updater, ipset_mgr):
        super(ProfileRules, self).__init__(qualifier=profile_id)
        assert profile_id is not None

        self.iptables_generator = iptables_generator
        self.id = profile_id
        self.ip_version = ip_version
        self._ipset_mgr = ipset_mgr
        self._iptables_updater = iptables_updater
        self._ipset_refs = RefHelper(self, ipset_mgr, self._on_ipsets_acquired)

        # Latest profile update - a profile dictionary.
        self._pending_profile = None
        # Currently-programmed profile dictionary.
        self._profile = None
        # The IDs of the tags and selector ipsets it requires.
        self._required_ipsets = set()

        # State flags.
        self._notified_ready = False
        self._cleaned_up = False
        self._dead = False
        self._dirty = True

    @actor_message()
    def on_profile_update(self, profile, force_reprogram=False):
        """
        Update the programmed iptables configuration with the new
        profile.

        :param dict[str]|NoneType profile: Dictionary of all profile data or
            None if profile is to be deleted.
        """
        _log.debug("%s: Profile update: %s", self, profile)
        assert not self._dead, "Shouldn't receive updates after we're dead."
        self._pending_profile = profile
        self._dirty |= force_reprogram

    @actor_message()
    def on_unreferenced(self):
        """
        Called to tell us that this profile is no longer needed.
        """
        # Flag that we're dead and then let finish_msg_batch() do the cleanup.
        self._dead = True

    def _on_ipsets_acquired(self):
        """
        Callback from the RefHelper once it's acquired all the ipsets we
        need.

        This is called from an actor_message on our greenlet.
        """
        # Nothing to do here, if this is being called then we're already in
        # a message batch so _finish_msg_batch() will get called next.
        _log.info("All required ipsets acquired.")

    def _finish_msg_batch(self, batch, results):
        # Due to dependency management in IptablesUpdater, we don't need to
        # worry about programming the dataplane before notifying so do it on
        # this common code path.
        if not self._notified_ready:
            self._notify_ready()
            self._notified_ready = True

        if self._dead:
            # Only want to clean up once.  Note: we can get here a second time
            # if we had a pending ipset incref in-flight when we were asked
            # to clean up.
            if not self._cleaned_up:
                try:
                    _log.info("%s unreferenced, removing our chains", self)
                    self._delete_chains()
                    self._ipset_refs.discard_all()
                    self._ipset_refs = None  # Break ref cycle.
                    self._profile = None
                    self._pending_profile = None
                finally:
                    self._cleaned_up = True
                    self._notify_cleanup_complete()
        else:
            if self._pending_profile != self._profile:
                _log.debug("Profile data changed, updating ipset references.")
                # Make sure that all the new tags and selectors are active.
                # We can't discard unneeded ones until we've updated iptables.
                new_tags_and_sels = extract_tags_and_selectors_from_profile(
                    self._pending_profile
                )
                for tag_or_sel in new_tags_and_sels:
                    _log.debug("Requesting ipset for tag %s", tag_or_sel)
                    # Note: acquire_ref() is a no-op if already acquired.
                    self._ipset_refs.acquire_ref(tag_or_sel)

                self._dirty = True
                self._profile = self._pending_profile
#.........这里部分代码省略.........

# 需要导入模块: from calico.felix.refcount import RefHelper [as 别名]
# 或者: from calico.felix.refcount.RefHelper import discard_all [as 别名]

#.........这里部分代码省略.........
            # _iptables_in_sync flag.
            _log.debug("iptables is out-of-sync, trying to update it")
            if self._admin_up:
                _log.info("%s is 'active', (re)programming chains.", self)
                self._update_chains()
            elif self._chains_programmed:
                # No longer active but our chains are still in place.  Remove
                # them.
                _log.info("%s is not 'active', removing chains.", self)
                self._remove_chains()

        if not self._device_in_sync and self._iface_name:
            # Try to update the device configuration.  If successful, will set
            # the _device_in_sync flag.
            if self._admin_up:
                # Endpoint is supposed to be live, try to configure it.
                _log.debug("Device is out-of-sync, trying to configure it")
                self._configure_interface()
            else:
                # We've been deleted, de-configure the interface.
                _log.debug("Device is out-of-sync, trying to de-configure it")
                self._deconfigure_interface()

        if self._removed_ips:
            # Some IPs have been removed, clean up conntrack.
            _log.debug("Some IPs were removed, cleaning up conntrack")
            self._clean_up_conntrack_entries()

        if self._unreferenced:
            # Endpoint is being removed, clean up...
            _log.debug("Cleaning up after endpoint unreferenced")
            self.dispatch_chains.on_endpoint_removed(self._iface_name,
                                                     async=True)
            self._rules_ref_helper.discard_all()
            self._notify_cleanup_complete()
            self._cleaned_up = True
        elif not self._added_to_dispatch_chains and self._iface_name:
            # This must be the first batch, add ourself to the dispatch chains.
            _log.debug("Adding endpoint to dispatch chain")
            self.dispatch_chains.on_endpoint_added(self._iface_name,
                                                   async=True)
            self._added_to_dispatch_chains = True

        # If changed, report our status back to the datastore.
        self._maybe_update_status()

    def _maybe_update_status(self):
        if not self.config.REPORT_ENDPOINT_STATUS:
            _log.debug("Status reporting disabled. Not reporting status.")
            return

        status, reason = self.oper_status()

        if self._unreferenced or status != self._last_status:
            _log.info("%s: updating status to %s", reason, status)
            if self._unreferenced:
                _log.debug("Unreferenced, reporting status = None")
                status_dict = None
            else:
                _log.debug("Endpoint oper state changed to %s", status)
                status_dict = {"status": status}
            self.status_reporter.on_endpoint_status_changed(
                self.combined_id,
                self.ip_type,
                status_dict,
                async=True,

# 需要导入模块: from calico.felix.refcount import RefHelper [as 别名]
# 或者: from calico.felix.refcount.RefHelper import discard_all [as 别名]
class ProfileRules(RefCountedActor):
    """
    Actor that owns the per-profile rules chains.
    """
    def __init__(self, profile_id, ip_version, iptables_updater, ipset_mgr):
        super(ProfileRules, self).__init__(qualifier=profile_id)
        assert profile_id is not None

        self.id = profile_id
        self.ip_version = ip_version
        self.ipset_mgr = ipset_mgr
        self._iptables_updater = iptables_updater
        self.notified_ready = False

        self.ipset_refs = RefHelper(self, ipset_mgr, self._maybe_update)

        self._profile = None
        """
        :type dict|None: filled in by first update.  Reset to None on delete.
        """
        self.dead = False

        self.chain_names = {
            "inbound": profile_to_chain_name("inbound", profile_id),
            "outbound": profile_to_chain_name("outbound", profile_id),
        }
        _log.info("Profile %s has chain names %s",
                  profile_id, self.chain_names)

    @actor_message()
    def on_profile_update(self, profile):
        """
        Update the programmed iptables configuration with the new
        profile.
        """
        _log.debug("%s: Profile update: %s", self, profile)
        assert profile is None or profile["id"] == self.id
        assert not self.dead, "Shouldn't receive updates after we're dead."

        old_tags = extract_tags_from_profile(self._profile)
        new_tags = extract_tags_from_profile(profile)

        removed_tags = old_tags - new_tags
        added_tags = new_tags - old_tags
        for tag in removed_tags:
            _log.debug("Queueing ipset for tag %s for decref", tag)
            self.ipset_refs.discard_ref(tag)
        for tag in added_tags:
            _log.debug("Requesting ipset for tag %s", tag)
            self.ipset_refs.acquire_ref(tag)

        self._profile = profile
        self._maybe_update()

    def _maybe_update(self):
        if self.dead:
            _log.info("Not updating: profile is dead.")
        elif not self.ipset_refs.ready:
            _log.info("Can't program rules %s yet, waiting on ipsets",
                      self.id)
        else:
            _log.info("Ready to program rules for %s", self.id)
            self._update_chains()

    @actor_message()
    def on_unreferenced(self):
        """
        Called to tell us that this profile is no longer needed.  Removes
        our iptables configuration.
        """
        try:
            _log.info("%s unreferenced, removing our chains", self)
            self.dead = True
            chains = []
            for direction in ["inbound", "outbound"]:
                chain_name = self.chain_names[direction]
                chains.append(chain_name)
            self._iptables_updater.delete_chains(chains, async=False)
            self.ipset_refs.discard_all()
            self.ipset_refs = None # Break ref cycle.
            self._profile = None
        finally:
            self._notify_cleanup_complete()

    def _update_chains(self):
        """
        Updates the chains in the dataplane.
        """
        _log.info("%s Programming iptables with our chains.", self)
        updates = {}
        for direction in ("inbound", "outbound"):
            _log.debug("Updating %s chain for profile %s", direction,
                       self.id)
            new_profile = self._profile or {}
            _log.debug("Profile %s: %s", self.id, self._profile)
            rules_key = "%s_rules" % direction
            new_rules = new_profile.get(rules_key, [])
            chain_name = self.chain_names[direction]
            tag_to_ip_set_name = {}
            for tag, ipset in self.ipset_refs.iteritems():
#.........这里部分代码省略.........

# 需要导入模块: from calico.felix.refcount import RefHelper [as 别名]
# 或者: from calico.felix.refcount.RefHelper import discard_all [as 别名]
class ProfileRules(RefCountedActor):
    """
    Actor that owns the per-profile rules chains.
    """
    def __init__(self, profile_id, ip_version, iptables_updater, ipset_mgr):
        super(ProfileRules, self).__init__(qualifier=profile_id)
        assert profile_id is not None

        self.id = profile_id
        self.ip_version = ip_version
        self._ipset_mgr = ipset_mgr
        self._iptables_updater = iptables_updater
        self._ipset_refs = RefHelper(self, ipset_mgr, self._on_ipsets_acquired)

        # Latest profile update.
        self._pending_profile = None
        # Currently-programmed profile.
        self._profile = None

        # State flags.
        self._notified_ready = False
        self._cleaned_up = False
        self._dead = False
        self._dirty = True

        self.chain_names = {
            "inbound": profile_to_chain_name("inbound", profile_id),
            "outbound": profile_to_chain_name("outbound", profile_id),
        }
        _log.info("Profile %s has chain names %s",
                  profile_id, self.chain_names)

    @actor_message()
    def on_profile_update(self, profile):
        """
        Update the programmed iptables configuration with the new
        profile.
        """
        _log.debug("%s: Profile update: %s", self, profile)
        assert not self._dead, "Shouldn't receive updates after we're dead."
        self._pending_profile = profile

    @actor_message()
    def on_unreferenced(self):
        """
        Called to tell us that this profile is no longer needed.
        """
        # Flag that we're dead and then let finish_msg_batch() do the cleanup.
        self._dead = True

    def _on_ipsets_acquired(self):
        """
        Callback from the RefHelper once it's acquired all the ipsets we
        need.

        This is called from an actor_message on our greenlet.
        """
        # Nothing to do here, if this is being called then we're already in
        # a message batch so _finish_msg_batch() will get called next.
        _log.info("All required ipsets acquired.")

    def _finish_msg_batch(self, batch, results):
        # Due to dependency management in IptablesUpdater, we don't need to
        # worry about programming the dataplane before notifying so do it on
        # this common code path.
        if not self._notified_ready:
            self._notify_ready()
            self._notified_ready = True

        if self._dead:
            # Only want to clean up once.  Note: we can get here a second time
            # if we had a pending ipset incref in-flight when we were asked
            # to clean up.
            if not self._cleaned_up:
                try:
                    _log.info("%s unreferenced, removing our chains", self)
                    chains = set(self.chain_names.values())
                    # Need to block here: have to wait for chains to be deleted
                    # before we can decref our ipsets.
                    self._iptables_updater.delete_chains(chains, async=False)
                    self._ipset_refs.discard_all()
                    self._ipset_refs = None # Break ref cycle.
                    self._profile = None
                    self._pending_profile = None
                finally:
                    self._cleaned_up = True
                    self._notify_cleanup_complete()
        else:
            if self._pending_profile != self._profile:
                _log.debug("Profile data changed, updating ipset references.")
                old_tags = extract_tags_from_profile(self._profile)
                new_tags = extract_tags_from_profile(self._pending_profile)
                removed_tags = old_tags - new_tags
                added_tags = new_tags - old_tags
                for tag in removed_tags:
                    _log.debug("Queueing ipset for tag %s for decref", tag)
                    self._ipset_refs.discard_ref(tag)
                for tag in added_tags:
                    _log.debug("Requesting ipset for tag %s", tag)
                    self._ipset_refs.acquire_ref(tag)
#.........这里部分代码省略.........