Python db_replicator.ReplicatorRpc类代码示例

 def __init__(self, conf, logger=None):
     self.logger = logger or get_logger(conf, log_route='container-server')
     self.log_requests = config_true_value(conf.get('log_requests', 'true'))
     self.root = conf.get('devices', '/srv/node')
     self.mount_check = config_true_value(conf.get('mount_check', 'true'))
     self.node_timeout = int(conf.get('node_timeout', 3))
     self.conn_timeout = float(conf.get('conn_timeout', 0.5))
     replication_server = conf.get('replication_server', None)
     if replication_server is not None:
         replication_server = config_true_value(replication_server)
     self.replication_server = replication_server
     #: ContainerSyncCluster instance for validating sync-to values.
     self.realms_conf = ContainerSyncRealms(
         os.path.join(
             conf.get('swift_dir', '/etc/swift'),
             'container-sync-realms.conf'),
         self.logger)
     #: The list of hosts we're allowed to send syncs to. This can be
     #: overridden by data in self.realms_conf
     self.allowed_sync_hosts = [
         h.strip()
         for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',')
         if h.strip()]
     self.replicator_rpc = ReplicatorRpc(
         self.root, DATADIR, ContainerBroker, self.mount_check,
         logger=self.logger)
     self.auto_create_account_prefix = \
         conf.get('auto_create_account_prefix') or '.'
     if config_true_value(conf.get('allow_versions', 'f')):
         self.save_headers.append('x-versions-location')
     swift.common.db.DB_PREALLOCATION = \
         config_true_value(conf.get('db_preallocation', 'f'))

 def __init__(self, conf, logger=None):
     super(AccountController, self).__init__(conf)
     self.logger = logger or get_logger(conf, log_route="account-server")
     self.log_requests = config_true_value(conf.get("log_requests", "true"))
     self.root = conf.get("devices", "/srv/node")
     self.mount_check = config_true_value(conf.get("mount_check", "true"))
     self.replicator_rpc = ReplicatorRpc(self.root, DATADIR, AccountBroker, self.mount_check, logger=self.logger)
     self.auto_create_account_prefix = conf.get("auto_create_account_prefix") or "."
     swift.common.db.DB_PREALLOCATION = config_true_value(conf.get("db_preallocation", "f"))

 def __init__(self, conf, logger=None):
     super(AccountController, self).__init__(conf)
     self.logger = logger or get_logger(conf, log_route='account-server')
     self.log_requests = config_true_value(conf.get('log_requests', 'true'))
     self.root = conf.get('devices', '/srv/node')
     self.mount_check = config_true_value(conf.get('mount_check', 'true'))
     self.replicator_rpc = ReplicatorRpc(self.root, DATADIR, AccountBroker,
                                         self.mount_check,
                                         logger=self.logger)
     self.auto_create_account_prefix = \
         conf.get('auto_create_account_prefix') or '.'
     swift.common.db.DB_PREALLOCATION = \
         config_true_value(conf.get('db_preallocation', 'f'))

 def __init__(self, conf):
     self.logger = get_logger(conf, log_route='account-server')
     self.root = conf.get('devices', '/srv/node')
     self.mount_check = config_true_value(conf.get('mount_check', 'true'))
     replication_server = conf.get('replication_server', None)
     if replication_server is not None:
         replication_server = config_true_value(replication_server)
     self.replication_server = replication_server
     self.replicator_rpc = ReplicatorRpc(self.root, DATADIR, AccountBroker,
                                         self.mount_check,
                                         logger=self.logger)
     self.auto_create_account_prefix = \
         conf.get('auto_create_account_prefix') or '.'
     swift.common.db.DB_PREALLOCATION = \
         config_true_value(conf.get('db_preallocation', 'f'))

    def __init__(self, conf, logger=None):
	#logging.basicConfig(filename="/home/ceph-w/swift/swift/account/log", level=logging.info)
	#logging.info("...account server init...")
        self.logger = logger or get_logger(conf, log_route='account-server')
        self.root = conf.get('devices', '/srv/node')
        self.mount_check = config_true_value(conf.get('mount_check', 'true'))
        replication_server = conf.get('replication_server', None)
        if replication_server is not None:
            replication_server = config_true_value(replication_server)
        self.replication_server = replication_server
        self.replicator_rpc = ReplicatorRpc(self.root, DATADIR, AccountBroker,
                                            self.mount_check,
                                            logger=self.logger)
        self.auto_create_account_prefix = \
            conf.get('auto_create_account_prefix') or '.'
        swift.common.db.DB_PREALLOCATION = \
            config_true_value(conf.get('db_preallocation', 'f'))

class AccountController(BaseStorageServer):
    """WSGI controller for the account server."""

    def __init__(self, conf, logger=None):
        super(AccountController, self).__init__(conf)
        self.logger = logger or get_logger(conf, log_route="account-server")
        self.log_requests = config_true_value(conf.get("log_requests", "true"))
        self.root = conf.get("devices", "/srv/node")
        self.mount_check = config_true_value(conf.get("mount_check", "true"))
        self.replicator_rpc = ReplicatorRpc(self.root, DATADIR, AccountBroker, self.mount_check, logger=self.logger)
        self.auto_create_account_prefix = conf.get("auto_create_account_prefix") or "."
        swift.common.db.DB_PREALLOCATION = config_true_value(conf.get("db_preallocation", "f"))

    def _get_account_broker(self, drive, part, account, **kwargs):
        hsh = hash_path(account)
        db_dir = storage_directory(DATADIR, part, hsh)
        db_path = os.path.join(self.root, drive, db_dir, hsh + ".db")
        kwargs.setdefault("account", account)
        kwargs.setdefault("logger", self.logger)
        return AccountBroker(db_path, **kwargs)

    def _deleted_response(self, broker, req, resp, body=""):
        # We are here since either the account does not exist or
        # it exists but marked for deletion.
        headers = {}
        # Try to check if account exists and is marked for deletion
        try:
            if broker.is_status_deleted():
                # Account does exist and is marked for deletion
                headers = {"X-Account-Status": "Deleted"}
        except DatabaseConnectionError:
            # Account does not exist!
            pass
        return resp(request=req, headers=headers, charset="utf-8", body=body)

    @public
    @timing_stats()
    def DELETE(self, req):
        """Handle HTTP DELETE request."""
        drive, part, account = split_and_validate_path(req, 3)
        if self.mount_check and not check_mount(self.root, drive):
            return HTTPInsufficientStorage(drive=drive, request=req)
        req_timestamp = valid_timestamp(req)
        broker = self._get_account_broker(drive, part, account)
        if broker.is_deleted():
            return self._deleted_response(broker, req, HTTPNotFound)
        broker.delete_db(req_timestamp.internal)
        return self._deleted_response(broker, req, HTTPNoContent)

    @public
    @timing_stats()
    def PUT(self, req):
        """Handle HTTP PUT request."""
        drive, part, account, container = split_and_validate_path(req, 3, 4)
        if self.mount_check and not check_mount(self.root, drive):
            return HTTPInsufficientStorage(drive=drive, request=req)
        if container:  # put account container
            if "x-timestamp" not in req.headers:
                timestamp = Timestamp(time.time())
            else:
                timestamp = valid_timestamp(req)
            pending_timeout = None
            container_policy_index = req.headers.get("X-Backend-Storage-Policy-Index", 0)
            if "x-trans-id" in req.headers:
                pending_timeout = 3
            broker = self._get_account_broker(drive, part, account, pending_timeout=pending_timeout)
            if account.startswith(self.auto_create_account_prefix) and not os.path.exists(broker.db_file):
                try:
                    broker.initialize(timestamp.internal)
                except DatabaseAlreadyExists:
                    pass
            if req.headers.get("x-account-override-deleted", "no").lower() != "yes" and broker.is_deleted():
                return HTTPNotFound(request=req)
            broker.put_container(
                container,
                req.headers["x-put-timestamp"],
                req.headers["x-delete-timestamp"],
                req.headers["x-object-count"],
                req.headers["x-bytes-used"],
                container_policy_index,
            )
            if req.headers["x-delete-timestamp"] > req.headers["x-put-timestamp"]:
                return HTTPNoContent(request=req)
            else:
                return HTTPCreated(request=req)
        else:  # put account
            timestamp = valid_timestamp(req)
            broker = self._get_account_broker(drive, part, account)
            if not os.path.exists(broker.db_file):
                try:
                    broker.initialize(timestamp.internal)
                    created = True
                except DatabaseAlreadyExists:
                    created = False
            elif broker.is_status_deleted():
                return self._deleted_response(broker, req, HTTPForbidden, body="Recently deleted")
            else:
                created = broker.is_deleted()
                broker.update_put_timestamp(timestamp.internal)
                if broker.is_deleted():
#.........这里部分代码省略.........

class AccountController(object):
    """WSGI controller for the account server."""

    def __init__(self, conf, logger=None):
        self.logger = logger or get_logger(conf, log_route='account-server')
        self.log_requests = config_true_value(conf.get('log_requests', 'true'))
        self.root = conf.get('devices', '/srv/node')
        self.mount_check = config_true_value(conf.get('mount_check', 'true'))
        replication_server = conf.get('replication_server', None)
        if replication_server is not None:
            replication_server = config_true_value(replication_server)
        self.replication_server = replication_server
        self.replicator_rpc = ReplicatorRpc(self.root, DATADIR, AccountBroker,
                                            self.mount_check,
                                            logger=self.logger)
        self.auto_create_account_prefix = \
            conf.get('auto_create_account_prefix') or '.'
        swift.common.db.DB_PREALLOCATION = \
            config_true_value(conf.get('db_preallocation', 'f'))

    def _get_account_broker(self, drive, part, account, **kwargs):
        hsh = hash_path(account)
        db_dir = storage_directory(DATADIR, part, hsh)
        db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
        kwargs.setdefault('account', account)
        kwargs.setdefault('logger', self.logger)
        return AccountBroker(db_path, **kwargs)

    def _deleted_response(self, broker, req, resp, body=''):
        # We are here since either the account does not exist or
        # it exists but marked for deletion.
        headers = {}
        # Try to check if account exists and is marked for deletion
        try:
            if broker.is_status_deleted():
                # Account does exist and is marked for deletion
                headers = {'X-Account-Status': 'Deleted'}
        except DatabaseConnectionError:
            # Account does not exist!
            pass
        return resp(request=req, headers=headers, charset='utf-8', body=body)

    @public
    @timing_stats()
    def DELETE(self, req):
        """Handle HTTP DELETE request."""
        drive, part, account = split_and_validate_path(req, 3)
        if self.mount_check and not check_mount(self.root, drive):
            return HTTPInsufficientStorage(drive=drive, request=req)
        req_timestamp = valid_timestamp(req)
        broker = self._get_account_broker(drive, part, account)
        if broker.is_deleted():
            return self._deleted_response(broker, req, HTTPNotFound)
        broker.delete_db(req_timestamp.internal)
        return self._deleted_response(broker, req, HTTPNoContent)

    @public
    @timing_stats()
    def PUT(self, req):
        """Handle HTTP PUT request."""
        drive, part, account, container = split_and_validate_path(req, 3, 4)
        if self.mount_check and not check_mount(self.root, drive):
            return HTTPInsufficientStorage(drive=drive, request=req)
        if container:   # put account container
            if 'x-timestamp' not in req.headers:
                timestamp = Timestamp(time.time())
            else:
                timestamp = valid_timestamp(req)
            pending_timeout = None
            container_policy_index = \
                req.headers.get('X-Backend-Storage-Policy-Index', 0)
            if 'x-trans-id' in req.headers:
                pending_timeout = 3
            broker = self._get_account_broker(drive, part, account,
                                              pending_timeout=pending_timeout)
            if account.startswith(self.auto_create_account_prefix) and \
                    not os.path.exists(broker.db_file):
                try:
                    broker.initialize(timestamp.internal)
                except DatabaseAlreadyExists:
                    pass
            if req.headers.get('x-account-override-deleted', 'no').lower() != \
                    'yes' and broker.is_deleted():
                return HTTPNotFound(request=req)
            broker.put_container(container, req.headers['x-put-timestamp'],
                                 req.headers['x-delete-timestamp'],
                                 req.headers['x-object-count'],
                                 req.headers['x-bytes-used'],
                                 container_policy_index)
            if req.headers['x-delete-timestamp'] > \
                    req.headers['x-put-timestamp']:
                return HTTPNoContent(request=req)
            else:
                return HTTPCreated(request=req)
        else:   # put account
            timestamp = valid_timestamp(req)
            broker = self._get_account_broker(drive, part, account)
            if not os.path.exists(broker.db_file):
                try:
                    broker.initialize(timestamp.internal)
#.........这里部分代码省略.........

class ContainerController(object):
    """WSGI Controller for the container server."""

    # Ensure these are all lowercase
    save_headers = ['x-container-read', 'x-container-write',
                    'x-container-sync-key', 'x-container-sync-to']

    def __init__(self, conf, logger=None):
        self.logger = logger or get_logger(conf, log_route='container-server')
        self.log_requests = config_true_value(conf.get('log_requests', 'true'))
        self.root = conf.get('devices', '/srv/node')
        self.mount_check = config_true_value(conf.get('mount_check', 'true'))
        self.node_timeout = int(conf.get('node_timeout', 3))
        self.conn_timeout = float(conf.get('conn_timeout', 0.5))
        replication_server = conf.get('replication_server', None)
        if replication_server is not None:
            replication_server = config_true_value(replication_server)
        self.replication_server = replication_server
        #: ContainerSyncCluster instance for validating sync-to values.
        self.realms_conf = ContainerSyncRealms(
            os.path.join(
                conf.get('swift_dir', '/etc/swift'),
                'container-sync-realms.conf'),
            self.logger)
        #: The list of hosts we're allowed to send syncs to. This can be
        #: overridden by data in self.realms_conf
        self.allowed_sync_hosts = [
            h.strip()
            for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',')
            if h.strip()]
        self.replicator_rpc = ReplicatorRpc(
            self.root, DATADIR, ContainerBroker, self.mount_check,
            logger=self.logger)
        self.auto_create_account_prefix = \
            conf.get('auto_create_account_prefix') or '.'
        if config_true_value(conf.get('allow_versions', 'f')):
            self.save_headers.append('x-versions-location')
        swift.common.db.DB_PREALLOCATION = \
            config_true_value(conf.get('db_preallocation', 'f'))

    def _get_container_broker(self, drive, part, account, container, **kwargs):
        """
        Get a DB broker for the container.

        :param drive: drive that holds the container
        :param part: partition the container is in
        :param account: account name
        :param container: container name
        :returns: ContainerBroker object
        """
        hsh = hash_path(account, container)
        db_dir = storage_directory(DATADIR, part, hsh)
        db_path = os.path.join(self.root, drive, db_dir, hsh + '.db')
        kwargs.setdefault('account', account)
        kwargs.setdefault('container', container)
        kwargs.setdefault('logger', self.logger)
        return ContainerBroker(db_path, **kwargs)

    def account_update(self, req, account, container, broker):
        """
        Update the account server(s) with latest container info.

        :param req: swob.Request object
        :param account: account name
        :param container: container name
        :param broker: container DB broker object
        :returns: if all the account requests return a 404 error code,
                  HTTPNotFound response object,
                  if the account cannot be updated due to a malformed header,
                  an HTTPBadRequest response object,
                  otherwise None.
        """
        account_hosts = [h.strip() for h in
                         req.headers.get('X-Account-Host', '').split(',')]
        account_devices = [d.strip() for d in
                           req.headers.get('X-Account-Device', '').split(',')]
        account_partition = req.headers.get('X-Account-Partition', '')

        if len(account_hosts) != len(account_devices):
            # This shouldn't happen unless there's a bug in the proxy,
            # but if there is, we want to know about it.
            self.logger.error(_('ERROR Account update failed: different  '
                                'numbers of hosts and devices in request: '
                                '"%s" vs "%s"') %
                               (req.headers.get('X-Account-Host', ''),
                                req.headers.get('X-Account-Device', '')))
            return HTTPBadRequest(req=req)

        if account_partition:
            updates = zip(account_hosts, account_devices)
        else:
            updates = []

        account_404s = 0

        for account_host, account_device in updates:
            account_ip, account_port = account_host.rsplit(':', 1)
            new_path = '/' + '/'.join([account, container])
            info = broker.get_info()
            account_headers = HeaderKeyDict({
#.........这里部分代码省略.........