Python etcd.EtcdAlreadyExist方法代码示例

# 需要导入模块: import etcd [as 别名]
# 或者: from etcd import EtcdAlreadyExist [as 别名]
def _compare_and_swap_block(self, block):
        """
        Write the block using an atomic Compare-and-swap.
        """

        # If the block has a db_result, CAS against that.
        if block.db_result is not None:
            _log.debug("CAS Update block %s", block)
            try:
                self.etcd_client.update(block.update_result())
            except EtcdCompareFailed:
                raise CASError(str(block.cidr))
        else:
            _log.debug("CAS Write new block %s", block)
            key = _block_datastore_key(block.cidr)
            value = block.to_json()
            try:
                self.etcd_client.write(key, value, prevExist=False)
            except EtcdAlreadyExist:
                raise CASError(str(block.cidr)) 

# 需要导入模块: import etcd [as 别名]
# 或者: from etcd import EtcdAlreadyExist [as 别名]
def atom_update_key(self, key_type, key, func, *args, **kwargs):
        """Wrap a function to atomically update a key

        Read the current value of the key and pass it to the wrapped
        function which returns the updated value. Then, try to
        update the value with compare and swap and restart the whole
        process if there was a race.

        """
        while True:
            try:
                value, index = self.read_key_index(key_type, key,
                                                   realindex=True)
                nargs = args + (value,)
                new_value, ret = func(*nargs, **kwargs)

                logging.debug(
                    "Trying atomic update \"%s\" for \"%s\" ",
                        str(value).strip(),
                        str(new_value).strip())

                if value is None:
                    if new_value is None:
                        return ret
                    else:
                        self.write_key_new(key_type, key, new_value)
                else:
                    self.write_key_index(key_type, key, new_value,
                                         index)

                return ret
            except ( etcd.EtcdCompareFailed,
                     etcd.EtcdKeyNotFound,
                     etcd.EtcdAlreadyExist ):
                logging.debug("Retrying atomic update") 

# 需要导入模块: import etcd [as 别名]
# 或者: from etcd import EtcdAlreadyExist [as 别名]
def _get_lock(self, lock_key, ttl=3, timeout=4):
        if timeout is not None:
            timeout_time = time.time() + timeout  # 5 minutes from now
        while True:
            try:
                etcd_client.write(lock_key, 'lock', prevExist=False, ttl=ttl)
                return True
            except etcd.EtcdAlreadyExist:
                if timeout is None or time.time() > timeout_time:
                    raise UnableToLockResource("%s already locked" % lock_key, {
                        "lock_key": lock_key,
                        "ttl": ttl})
                else:
                    time.sleep(0.2) 

# 需要导入模块: import etcd [as 别名]
# 或者: from etcd import EtcdAlreadyExist [as 别名]
def test_locked(monkeypatch, package_b64blob):

    def write(path, data, prevExist, ttl=None):
        raise etcd.EtcdAlreadyExist

    def read(path):
        assert path == ""
        return True
    monkeypatch.setattr("appr.models.kv.models_index_base.ModelsIndexBase.get_lock.im_func.__defaults__", (3, 0.1))

    monkeypatch.setattr("appr.models.kv.etcd.etcd_client.read", read)
    monkeypatch.setattr("appr.models.kv.etcd.etcd_client.write", write)
    with pytest.raises(UnableToLockResource):
        p = Package('a/b', "1.0.1", 'kpm', Blob("a/b", package_b64blob))
        p.save() 

# 需要导入模块: import etcd [as 别名]
# 或者: from etcd import EtcdAlreadyExist [as 别名]
def init_phase(self):
        """
        Initially, the rendezvous state is expected to be one of:

        1. empty (non-existent) - in this case we try to create a new one.
        2. joinable - we try to join it.
        3. final - we announce ourselves as waiting, and go into monitoring mode

        Any other state is considered transitional, and will be retried after
        a short delay.

        Returns:
            ``(rdzv_version, rank, world_size)``

        Raises:
            RendezvousClosedException - current rendezvous was/is closed
            EtcdRendezvousRetryableFailure - observed some intermediate
             state, which is best handled by retrying later
        """
        try:
            active_version = self.try_create_rendezvous()
            state = json.loads(active_version.value)
            log.info("New rendezvous state created: " + str(state))
        except etcd.EtcdAlreadyExist:
            active_version, state = self.get_rdzv_state()
            # Note: it is possible for above query to fail (etcd.EtcdKeyNotFound),
            # but this is ok for us - just means we'll restart from beginning.
            log.info("Observed existing rendezvous state: " + str(state))

        if state["status"] == "closed":
            raise RendezvousClosedException()

        if state["status"] == "joinable":
            return self.join_phase(state["version"])

        if state["status"] == "final":
            self.handle_existing_rendezvous(state["version"])
            raise EtcdRendezvousRetryImmediately()

        self.try_wait_for_state_change(etcd_index=active_version.etcd_index + 1)
        raise EtcdRendezvousRetryableFailure() 

# 需要导入模块: import etcd [as 别名]
# 或者: from etcd import EtcdAlreadyExist [as 别名]
def create_path_if_not_exists(self, full_path, ttl=None):
        try:
            self.client.write(
                key=full_path, value=None, dir=True, prevExist=False, ttl=ttl
            )
        except etcd.EtcdAlreadyExist:
            pass 

# 需要导入模块: import etcd [as 别名]
# 或者: from etcd import EtcdAlreadyExist [as 别名]
def store_extra_data(self, rdzv_version, key, value):
        node = self.get_path("/rdzv/v_{}/extra_data".format(rdzv_version))
        try:
            # If first time we are storing anything:
            extra_data = self.client.write(
                key=node, value=json.dumps({key: value}), prevExist=False
            )
            return
        except etcd.EtcdAlreadyExist:
            pass

        # CAS loop, to make sure we don't lose concurrent stores.
        while True:
            # We never delete extra_data. Failure here should be fatal, no special handling.
            extra_data = self.client.get(node)

            new_extra_data_value = json.loads(extra_data.value)
            new_extra_data_value[key] = value

            try:
                extra_data = self.client.test_and_set(
                    key=node,
                    value=json.dumps(new_extra_data_value),
                    prev_value=extra_data.value,
                )
                return
            except etcd.EtcdCompareFailed:
                log.info("Store extra_data CAS unsuccessful, retrying")
                time.sleep(0.1) 

# 需要导入模块: import etcd [as 别名]
# 或者: from etcd import EtcdAlreadyExist [as 别名]
def add(self, key, num: int) -> int:
        """
        Atomically increment a value by an integer amount. The integer is
        represented as a string using base 10. If key is not present,
        a default value of ``0`` will be assumed.

        Returns:
             the new (incremented) value


        """
        b64_key = self._encode(key)
        # c10d Store assumes value is an integer represented as a decimal string
        try:
            # Assume default value "0", if this key didn't yet:
            node = self.client.write(
                key=self.prefix + b64_key,
                value=self._encode(str(num)),  # i.e. 0 + num
                prevExist=False,
            )
            return int(self._decode(node.value))
        except etcd.EtcdAlreadyExist:
            pass

        while True:
            # Note: c10d Store does not have a method to delete keys, so we
            # can be sure it's still there.
            node = self.client.get(key=self.prefix + b64_key)
            new_value = self._encode(str(int(self._decode(node.value)) + num))
            try:
                node = self.client.test_and_set(
                    key=node.key, value=new_value, prev_value=node.value
                )
                return int(self._decode(node.value))
            except etcd.EtcdCompareFailed:
                cas_delay() 

# 需要导入模块: import etcd [as 别名]
# 或者: from etcd import EtcdAlreadyExist [as 别名]
def test_claim_block_affinity_already_owned(self):
        """
        Test _claim_block_affinity() when we already own the block

        Order of events
        1 Write host affinity
        2 Try to write the new block, but this fails.
        3 Read the block, check its affinity
        """

        block = _test_block_empty_v4()
        m_result0 = Mock(spec=EtcdResult)
        m_result0.value = block.to_json()

        # Reads at 3
        self.m_etcd_client.read.return_value = m_result0

        # Write at 1, 2
        self.m_etcd_client.write.side_effect = [None, EtcdAlreadyExist()]

        self.client._claim_block_affinity(block.host_affinity,
                                          block.cidr, IPAMConfig())

        key = _block_datastore_key(block.cidr)
        value = block.to_json()
        self.m_etcd_client.write.assert_has_calls([call(ANY, ""),
                                                   call(key, value,
                                                        prevExist=False)])
        self.m_etcd_client.read.assert_called_once_with(key, quorum=True) 

# 需要导入模块: import etcd [as 别名]
# 或者: from etcd import EtcdAlreadyExist [as 别名]
def test_claim_block_affinity_already_deleted(self):
        """
        Test _claim_block_affinity() when another process deletes the host
        affinity under our feet.
        This can occur when the block gets claimed by one host and a second
        host has two competing processes trying to claim the block. They can
        both try to clean up at the same time.

        Order of events
        1 Write host affinity
        2 Try to write the new block, but this fails
        3 Re-read the block, discover another host owns it
        4 Delete key from 1 but find it's already removed.
        """

        block = _test_block_empty_v4()
        m_result0 = Mock(spec=EtcdResult)
        m_result0.value = block.to_json()

        block_our_host = AllocationBlock(BLOCK_V4_1, "test_host2", False)

        # Reads at 3
        self.m_etcd_client.read.return_value = m_result0

        # Write at 1, 2
        self.m_etcd_client.write.side_effect = [None, EtcdAlreadyExist()]

        # Delete at 4
        self.m_etcd_client.delete.side_effect = [EtcdKeyNotFound()]

        with self.assertRaises(HostAffinityClaimedError):
            self.client._claim_block_affinity(block_our_host.host_affinity,
                                              block.cidr, IPAMConfig())

        key = _block_datastore_key(block_our_host.cidr)
        value = block_our_host.to_json()
        self.m_etcd_client.write.assert_has_calls([call(ANY, ""),
                                                   call(key, value,
                                                        prevExist=False)])
        self.m_etcd_client.read.assert_called_once_with(key, quorum=True)
        self.m_etcd_client.delete.assert_called_once_with(ANY) 

# 需要导入模块: import etcd [as 别名]
# 或者: from etcd import EtcdAlreadyExist [as 别名]
def test_compare_and_swap_handle_cas_error_new(self):
        """
        Test _compare_and_swap_handle hitting a CAS error on adding a new
        handle.
        """
        handle_id = "handle_id_1"
        block_cidr = BLOCK_V4_1
        amount = 10

        handle0 = AllocationHandle(handle_id)
        handle0.increment_block(block_cidr, amount)
        self.m_etcd_client.write.side_effect = EtcdAlreadyExist

        self.assertRaises(CASError, self.client._compare_and_swap_handle,
                          handle0) 

# 需要导入模块: import etcd [as 别名]
# 或者: from etcd import EtcdAlreadyExist [as 别名]
def _compare_and_swap_handle(self, handle):
        """
        Write the handle using an atomic Compare-and-swap.
        """
        # If the handle has a db_result, CAS against that.
        if handle.db_result is not None:
            _log.debug("Handle %s exists.", handle.handle_id)
            if handle.is_empty():
                # Handle is now empty.  Delete it instead of an update.
                _log.debug("Handle %s is empty.", handle.handle_id)
                key = _handle_datastore_key(handle.handle_id)
                try:
                    self.etcd_client.delete(
                        key,
                        prevIndex=handle.db_result.modifiedIndex)
                except EtcdCompareFailed:
                    raise CASError(handle.handle_id)
            else:
                _log.debug("Handle %s is not empty.", handle.handle_id)
                try:
                    self.etcd_client.update(handle.update_result())
                except EtcdCompareFailed:
                    raise CASError(handle.handle_id)
        else:
            _log.debug("CAS Write new handle %s", handle.handle_id)
            assert not handle.is_empty(), "Don't write empty handle."
            key = _handle_datastore_key(handle.handle_id)
            value = handle.to_json()
            try:
                self.etcd_client.write(key, value, prevExist=False)
            except EtcdAlreadyExist:
                raise CASError(handle.handle_id) 

# 需要导入模块: import etcd [as 别名]
# 或者: from etcd import EtcdAlreadyExist [as 别名]
def attempt_to_acquire_leader(self, permanent=False):
        try:
            return bool(self.retry(self._client.write,
                                   self.leader_path,
                                   self._name,
                                   ttl=None if permanent else self._ttl,
                                   prevExist=False))
        except etcd.EtcdAlreadyExist:
            logger.info('Could not take out TTL lock')
        except (RetryFailedError, etcd.EtcdException):
            pass
        return False 

# 需要导入模块: import etcd [as 别名]
# 或者: from etcd import EtcdAlreadyExist [as 别名]
def etcd_write(self, key, value, **kwargs):
    if key == '/service/exists/leader':
        raise etcd.EtcdAlreadyExist
    if key in ['/service/test/leader', '/patroni/test/leader'] and \
            (kwargs.get('prevValue') == 'foo' or not kwargs.get('prevExist', True)):
        return True
    raise etcd.EtcdException 

# 需要导入模块: import etcd [as 别名]
# 或者: from etcd import EtcdAlreadyExist [as 别名]
def __init__(
        self,
        endpoints,
        prefix,
        run_id,
        num_min_workers,
        num_max_workers,
        timeout,
        last_call_timeout,
        **kwargs,
    ):
        self._prefix = prefix
        self._run_id = run_id
        self._num_min_workers = num_min_workers
        self._num_max_workers = num_max_workers
        self._timeout = timeout
        self._last_call_timeout = last_call_timeout

        # For cleaning up TTL refresher threads (for ephemeral keys)
        self._lease_run_id_stop = None
        self._lease_this_rank_stop = None

        if not self._prefix.endswith("/"):
            self._prefix += "/"

        self.client = etcd.Client(host=endpoints, allow_reconnect=True, **kwargs)
        log.info("Etcd machines: " + str(self.client.machines))

        # Setup a permanent prefix dir, if didn't exist
        if self._prefix != "/":
            self.create_path_if_not_exists(self._prefix)

        # Lease a "sub-root" node specific to this job instance (run_id)
        self.create_path_if_not_exists(self.get_path(""), ttl=CONST_RUNID_SUBROOT_TTL)
        self._lease_run_id_stop = self.setup_lease_renewal(
            self.get_path(""), ttl=CONST_RUNID_SUBROOT_TTL
        )

        # Subdir for all rendezvous work
        self.create_path_if_not_exists(self.get_path("/rdzv"))

        # Create a rendezvous version counter, if doesn't exist
        try:
            self.client.write(
                key=self.get_path("/rdzv/version_counter"), value="0", prevExist=False
            )
        except etcd.EtcdAlreadyExist:
            pass