Python BindPortsCache.all_bind_ports_for_node方法代碼示例

# 需要導入模塊: from swift.common.storage_policy import BindPortsCache [as 別名]
# 或者: from swift.common.storage_policy.BindPortsCache import all_bind_ports_for_node [as 別名]
    def test_bind_ports_cache(self):
        test_policies = [StoragePolicy(0, 'aay', True),
                         StoragePolicy(1, 'bee', False),
                         StoragePolicy(2, 'cee', False)]

        my_ips = ['1.2.3.4', '2.3.4.5']
        other_ips = ['3.4.5.6', '4.5.6.7']
        bind_ip = my_ips[1]
        devs_by_ring_name1 = {
            'object': [  # 'aay'
                {'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[0],
                 'port': 6006},
                {'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[0],
                 'port': 6007},
                {'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[1],
                 'port': 6008},
                None,
                {'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[1],
                 'port': 6009}],
            'object-1': [  # 'bee'
                {'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[1],
                 'port': 6006},  # dupe
                {'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[0],
                 'port': 6010},
                {'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[1],
                 'port': 6011},
                {'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[1],
                 'port': 6012}],
            'object-2': [  # 'cee'
                {'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[0],
                 'port': 6010},  # on our IP and a not-us IP
                {'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[0],
                 'port': 6013},
                None,
                {'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[1],
                 'port': 6014},
                {'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[1],
                 'port': 6015}],
        }
        devs_by_ring_name2 = {
            'object': [  # 'aay'
                {'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[0],
                 'port': 6016},
                {'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[1],
                 'port': 6019}],
            'object-1': [  # 'bee'
                {'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[1],
                 'port': 6016},  # dupe
                {'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[1],
                 'port': 6022}],
            'object-2': [  # 'cee'
                {'id': 0, 'zone': 0, 'region': 1, 'ip': my_ips[0],
                 'port': 6020},
                {'id': 0, 'zone': 0, 'region': 1, 'ip': other_ips[1],
                 'port': 6025}],
        }
        ring_files = [ring_name + '.ring.gz'
                      for ring_name in sorted(devs_by_ring_name1)]

        def _fake_load(gz_path, stub_objs, metadata_only=False):
            return RingData(
                devs=stub_objs[os.path.basename(gz_path)[:-8]],
                replica2part2dev_id=[],
                part_shift=24)

        with mock.patch(
                'swift.common.storage_policy.RingData.load'
        ) as mock_ld, \
                patch_policies(test_policies), \
                mock.patch('swift.common.storage_policy.whataremyips') \
                as mock_whataremyips, \
                temptree(ring_files) as tempdir:
            mock_whataremyips.return_value = my_ips

            cache = BindPortsCache(tempdir, bind_ip)

            self.assertEqual([
                mock.call(bind_ip),
            ], mock_whataremyips.mock_calls)
            mock_whataremyips.reset_mock()

            mock_ld.side_effect = partial(_fake_load,
                                          stub_objs=devs_by_ring_name1)
            self.assertEqual(set([
                6006, 6008, 6011, 6010, 6014,
            ]), cache.all_bind_ports_for_node())
            self.assertEqual([
                mock.call(os.path.join(tempdir, ring_files[0]),
                          metadata_only=True),
                mock.call(os.path.join(tempdir, ring_files[1]),
                          metadata_only=True),
                mock.call(os.path.join(tempdir, ring_files[2]),
                          metadata_only=True),
            ], mock_ld.mock_calls)
            mock_ld.reset_mock()

            mock_ld.side_effect = partial(_fake_load,
                                          stub_objs=devs_by_ring_name2)
            self.assertEqual(set([
                6006, 6008, 6011, 6010, 6014,
#.........這裏部分代碼省略.........

# 需要導入模塊: from swift.common.storage_policy import BindPortsCache [as 別名]
# 或者: from swift.common.storage_policy.BindPortsCache import all_bind_ports_for_node [as 別名]
class ServersPerPortStrategy(object):
    """
    WSGI server management strategy object for an object-server with one listen
    port per unique local port in the storage policy rings.  The
    `servers_per_port` integer config setting determines how many workers are
    run per port.

    Used in :py:func:`run_wsgi`.

    :param dict conf: Server configuration dictionary.
    :param logger: The server's :py:class:`~swift.common.utils.LogAdaptor`
                   object.
    :param int servers_per_port: The number of workers to run per port.
    """

    def __init__(self, conf, logger, servers_per_port):
        self.conf = conf
        self.logger = logger
        self.servers_per_port = servers_per_port
        self.swift_dir = conf.get("swift_dir", "/etc/swift")
        self.ring_check_interval = int(conf.get("ring_check_interval", 15))
        self.port_pid_state = PortPidState(servers_per_port, logger)

        bind_ip = conf.get("bind_ip", "0.0.0.0")
        self.cache = BindPortsCache(self.swift_dir, bind_ip)

    def _reload_bind_ports(self):
        self.bind_ports = self.cache.all_bind_ports_for_node()

    def _bind_port(self, port):
        new_conf = self.conf.copy()
        new_conf["bind_port"] = port
        sock = get_socket(new_conf)
        self.port_pid_state.track_port(port, sock)

    def loop_timeout(self):
        """
        Return timeout before checking for reloaded rings.

        :returns: The time to wait for a child to exit before checking for
                  reloaded rings (new ports).
        """

        return self.ring_check_interval

    def bind_ports(self):
        """
        Bind one listen socket per unique local storage policy ring port.  Then
        do all the work of drop_privileges except the actual dropping of
        privileges (each forked-off worker will do that post-fork in
        :py:meth:`post_fork_hook`).
        """

        self._reload_bind_ports()
        for port in self.bind_ports:
            self._bind_port(port)

        # The workers strategy drops privileges here, which we obviously cannot
        # do if we want to support binding to low ports.  But we do want some
        # of the actions that drop_privileges did.
        try:
            os.setsid()
        except OSError:
            pass
        # In case you need to rmdir where you started the daemon:
        os.chdir("/")
        # Ensure files are created with the correct privileges:
        os.umask(0o22)

    def no_fork_sock(self):
        """
        This strategy does not support running in the foreground.
        """

        pass

    def new_worker_socks(self):
        """
        Yield a sequence of (socket, server_idx) tuples for each server which
        should be forked-off and started.

        Any sockets for "orphaned" ports no longer in any ring will be closed
        (causing their associated workers to gracefully exit) after all new
        sockets have been yielded.

        The server_idx item for each socket will passed into the
        :py:meth:`log_sock_exit` and :py:meth:`register_worker_start` methods.
        """

        self._reload_bind_ports()
        desired_port_index_pairs = set((p, i) for p in self.bind_ports for i in range(self.servers_per_port))

        current_port_index_pairs = self.port_pid_state.port_index_pairs()

        if desired_port_index_pairs != current_port_index_pairs:
            # Orphan ports are ports which had object-server processes running,
            # but which no longer appear in the ring.  We'll kill them after we
            # start missing workers.
            orphan_port_index_pairs = current_port_index_pairs - desired_port_index_pairs

#.........這裏部分代碼省略.........

# 需要導入模塊: from swift.common.storage_policy import BindPortsCache [as 別名]
# 或者: from swift.common.storage_policy.BindPortsCache import all_bind_ports_for_node [as 別名]
    def test_bind_ports_cache(self):
        test_policies = [StoragePolicy(0, "aay", True), StoragePolicy(1, "bee", False), StoragePolicy(2, "cee", False)]

        my_ips = ["1.2.3.4", "2.3.4.5"]
        other_ips = ["3.4.5.6", "4.5.6.7"]
        bind_ip = my_ips[1]
        devs_by_ring_name1 = {
            "object": [  # 'aay'
                {"id": 0, "zone": 0, "region": 1, "ip": my_ips[0], "port": 6006},
                {"id": 0, "zone": 0, "region": 1, "ip": other_ips[0], "port": 6007},
                {"id": 0, "zone": 0, "region": 1, "ip": my_ips[1], "port": 6008},
                None,
                {"id": 0, "zone": 0, "region": 1, "ip": other_ips[1], "port": 6009},
            ],
            "object-1": [  # 'bee'
                {"id": 0, "zone": 0, "region": 1, "ip": my_ips[1], "port": 6006},  # dupe
                {"id": 0, "zone": 0, "region": 1, "ip": other_ips[0], "port": 6010},
                {"id": 0, "zone": 0, "region": 1, "ip": my_ips[1], "port": 6011},
                {"id": 0, "zone": 0, "region": 1, "ip": other_ips[1], "port": 6012},
            ],
            "object-2": [  # 'cee'
                {"id": 0, "zone": 0, "region": 1, "ip": my_ips[0], "port": 6010},  # on our IP and a not-us IP
                {"id": 0, "zone": 0, "region": 1, "ip": other_ips[0], "port": 6013},
                None,
                {"id": 0, "zone": 0, "region": 1, "ip": my_ips[1], "port": 6014},
                {"id": 0, "zone": 0, "region": 1, "ip": other_ips[1], "port": 6015},
            ],
        }
        devs_by_ring_name2 = {
            "object": [  # 'aay'
                {"id": 0, "zone": 0, "region": 1, "ip": my_ips[0], "port": 6016},
                {"id": 0, "zone": 0, "region": 1, "ip": other_ips[1], "port": 6019},
            ],
            "object-1": [  # 'bee'
                {"id": 0, "zone": 0, "region": 1, "ip": my_ips[1], "port": 6016},  # dupe
                {"id": 0, "zone": 0, "region": 1, "ip": other_ips[1], "port": 6022},
            ],
            "object-2": [  # 'cee'
                {"id": 0, "zone": 0, "region": 1, "ip": my_ips[0], "port": 6020},
                {"id": 0, "zone": 0, "region": 1, "ip": other_ips[1], "port": 6025},
            ],
        }
        ring_files = [ring_name + ".ring.gz" for ring_name in sorted(devs_by_ring_name1)]

        def _fake_load(gz_path, stub_objs, metadata_only=False):
            return RingData(devs=stub_objs[os.path.basename(gz_path)[:-8]], replica2part2dev_id=[], part_shift=24)

        with mock.patch("swift.common.storage_policy.RingData.load") as mock_ld, patch_policies(
            test_policies
        ), mock.patch("swift.common.storage_policy.whataremyips") as mock_whataremyips, temptree(ring_files) as tempdir:
            mock_whataremyips.return_value = my_ips

            cache = BindPortsCache(tempdir, bind_ip)

            self.assertEqual([mock.call(bind_ip)], mock_whataremyips.mock_calls)
            mock_whataremyips.reset_mock()

            mock_ld.side_effect = partial(_fake_load, stub_objs=devs_by_ring_name1)
            self.assertEqual(set([6006, 6008, 6011, 6010, 6014]), cache.all_bind_ports_for_node())
            self.assertEqual(
                [
                    mock.call(os.path.join(tempdir, ring_files[0]), metadata_only=True),
                    mock.call(os.path.join(tempdir, ring_files[1]), metadata_only=True),
                    mock.call(os.path.join(tempdir, ring_files[2]), metadata_only=True),
                ],
                mock_ld.mock_calls,
            )
            mock_ld.reset_mock()

            mock_ld.side_effect = partial(_fake_load, stub_objs=devs_by_ring_name2)
            self.assertEqual(set([6006, 6008, 6011, 6010, 6014]), cache.all_bind_ports_for_node())
            self.assertEqual([], mock_ld.mock_calls)

            # but when all the file mtimes are made different, it'll
            # reload
            for gz_file in [os.path.join(tempdir, n) for n in ring_files]:
                os.utime(gz_file, (88, 88))

            self.assertEqual(set([6016, 6020]), cache.all_bind_ports_for_node())
            self.assertEqual(
                [
                    mock.call(os.path.join(tempdir, ring_files[0]), metadata_only=True),
                    mock.call(os.path.join(tempdir, ring_files[1]), metadata_only=True),
                    mock.call(os.path.join(tempdir, ring_files[2]), metadata_only=True),
                ],
                mock_ld.mock_calls,
            )
            mock_ld.reset_mock()

            # Don't do something stupid like crash if a ring file is missing.
            os.unlink(os.path.join(tempdir, "object-2.ring.gz"))

            self.assertEqual(set([6016, 6020]), cache.all_bind_ports_for_node())
            self.assertEqual([], mock_ld.mock_calls)

        # whataremyips() is only called in the constructor
        self.assertEqual([], mock_whataremyips.mock_calls)