Python model.Vault类代码示例

    def test_vault_crud(self):
        vault_id = self.create_vault_id()

        v = Vault.get(vault_id)
        assert v is None

        v = Vault.create(vault_id)
        assert v is not None

        v.delete()

        v = Vault.get(vault_id)
        assert v is None

    def on_get(self, req, resp, vault_id):
        vault = Vault.get(vault_id)

        if not vault:
            logger.error('Vault [{0}] does not exist'.format(vault_id))
            raise errors.HTTPNotFound
        # NOTE(TheSriram): get_param(param) automatically returns None
        # if param is not present
        inmarker = req.get_param('marker')
        limit = req.get_param_as_int('limit') if req.get_param_as_int('limit') \
            else conf.api_configuration.default_returned_num

        # The +1 is to fetch one past the user's
        # requested limit so that we can determine
        # if the list was truncated or not
        files = vault.get_files(inmarker, limit + 1)

        responses = list(files)

        # Note: the list may not actually be truncated
        truncated = len(responses) == limit + 1

        outmarker = responses.pop().file_id if truncated else None

        if outmarker:
            query_args = {'marker': outmarker}
            query_args['limit'] = limit

            returl = set_qs_on_url(req.url, query_args)

            resp.set_header("X-Next-Batch", returl)

        resp.body = json.dumps([response.file_id for response in responses])

    def on_post(self, req, resp, vault_id):
        vault = Vault.get(vault_id)
        try:
            unpacked = msgpack.unpackb(req.stream.read())

            if not isinstance(unpacked, dict):
                raise TypeError

            else:
                block_ids = list(unpacked.keys())
                block_datas = list(unpacked.values())
                try:
                    retval, retblocks = vault.put_async_block(
                        block_ids,
                        block_datas)
                    if retval:
                        resp.status = falcon.HTTP_200
                        resp.body = json.dumps({block_id: storage_id
                                               for block_id, storage_id
                                               in retblocks})
                    else:
                        raise errors.HTTPInternalServerError('Block '
                                                            'Post Failed')
                    logger.info('blocks [{0}] added'.format(block_ids))
                except ValueError:
                    raise errors.HTTPPreconditionFailed('hash error')
        except (TypeError, ValueError):
            logger.error('Request Body not well formed '
                         'for posting multiple blocks to {0}'.format(vault_id))
            raise errors.HTTPBadRequestBody("Request Body not well formed")

    def on_get(self, req, resp, vault_id, file_id):
        """Fetches, re-assembles and streams a single
        file out of Deuce"""
        vault = Vault.get(vault_id)

        if not vault:
            logger.error('Vault [{0}] does not exist'.format(vault_id))
            raise errors.HTTPNotFound

        f = vault.get_file(file_id)

        if not f:
            logger.error('File [{0}] does not exist'.format(file_id))
            raise errors.HTTPNotFound

        if not f.finalized:
            raise errors.HTTPConflict('File not Finalized')

        block_gen = deuce.metadata_driver.create_file_block_generator(
            vault_id, file_id)

        block_ids = [block[0] for block in sorted(block_gen,
                                                  key=lambda block: block[1])]

        objs = vault.get_blocks_generator(block_ids)

        # NOTE(TheSriram): falcon 0.2.0 might fix this problem,
        # we should be able to set resp.stream to any file like
        # object instead of an iterator.
        resp.stream = (obj.read() for obj in objs)
        resp.status = falcon.HTTP_200
        resp.set_header('Content-Length', str(vault.get_file_length(file_id)))
        resp.content_type = 'application/octet-stream'

    def on_get(self, req, resp, vault_id):

        vault = Vault.get(vault_id)
        if not vault:
            logger.error('Vault [{0}] does not exist'.format(vault_id))
            raise errors.HTTPNotFound
        # NOTE(TheSriram): get_param(param) automatically returns None
        # if param is not present
        inmarker = req.get_param('marker')
        limit = req.get_param_as_int('limit') if req.get_param_as_int('limit') else \
            conf.api_configuration.default_returned_num

        # We actually fetch the user's requested
        # limit +1 to detect if the list is being
        # truncated or not.
        blocks = vault.get_blocks(inmarker, limit + 1)

        # List the blocks into JSON and return.
        # TODO: figure out a way to stream this back(??)
        responses = list(blocks)

        # Was the list truncated? See note above about +1
        truncated = len(responses) > 0 and len(responses) == limit + 1

        outmarker = responses.pop().metadata_block_id if truncated else None

        if outmarker:
            query_args = {'marker': outmarker}
            query_args['limit'] = limit
            returl = set_qs_on_url(req.url, query_args)
            resp.set_header("X-Next-Batch", returl)

        resp.body = json.dumps([response.metadata_block_id
                                for response in responses])

    def get_one(self, vault_id, file_id):
        """Fetches, re-assembles and streams a single
        file out of Deuce"""
        vault = Vault.get(vault_id)

        if not vault:
            logger.error('Vault [{0}] does not exist'.format(vault_id))
            abort(404, headers={"Transaction-ID":
                deuce.context.transaction.request_id})

        f = vault.get_file(file_id)

        if not f:
            logger.error('File [{0}] does not exist'.format(file_id))
            abort(404, headers={"Transaction-ID":
                deuce.context.transaction.request_id})

        if not f.finalized:
            abort(412, headers={"Transaction-ID":
                deuce.context.transaction.request_id})

        block_gen = deuce.metadata_driver.create_file_block_generator(
            vault_id, file_id)

        block_ids = [block[0] for block in sorted(block_gen,
            key=lambda block: block[1])]

        objs = vault.get_blocks_generator(block_ids)

        response.content_length = vault.get_file_length(file_id)
        response.body_file = FileCat(objs)
        response.status_code = 200

    def get_all(self, vault_id):

        vault = Vault.get(request.project_id, vault_id)

        if not vault:
            response.status_code = 404
            return

        inmarker = request.params.get('marker')

        limit = int(request.params.get('limit',
           conf.api_configuration.max_returned_num))

        # We actually fetch the user's requested
        # limit +1 to detect if the list is being
        # truncated or not.
        blocks = vault.get_blocks(inmarker, limit + 1)

        # List the blocks into JSON and return.
        # TODO: figure out a way to stream this back(??)
        resp = list(blocks)

        # Was the list truncated? See note above about +1
        truncated = len(resp) > 0 and len(resp) == limit + 1

        outmarker = resp.pop().block_id if truncated else None

        if outmarker:
            query_args = {'marker': outmarker}
            query_args['limit'] = limit
            returl = set_qs(request.url, query_args)
            response.headers["X-Next-Batch"] = returl

        return resp

    def post(self, vault_id, file_id=None):
        """Initializes a new file. The location of
        the new file is returned in the Location
        header
        """
        vault = Vault.get(vault_id)

        # caller tried to post to a vault that
        # does not exist
        if not vault:
            logger.error('Vault [{0}] does not exist'.format(vault_id))
            abort(400, headers={"Transaction-ID":
                deuce.context.transaction.request_id})

        # overload to use the same end-point for creating a new file
        # and assigning blocks to a file that is in progress
        if file_id is not None:
            return self._assign(vault, vault_id, file_id)

        file = vault.create_file()

        response.headers["Location"] = "files/%s" % file.file_id
        response.status_code = 201  # Created
        logger.info('File [{0}] created'.
            format(response.headers["Location"]))

    def on_delete(self, req, resp, vault_id, block_id):
        """Unregisters a block_id from a given vault_id in
        the storage and metadata
        """
        vault = Vault.get(vault_id)

        if not vault:
            logger.error('Vault [{0}] does not exist'.format(vault_id))
            resp.status = falcon.HTTP_404
            return

        try:
            response = vault.delete_block(vault_id, block_id)

        except ConstraintError as ex:
            logger.error(json.dumps(ex.args))
            raise errors.HTTPConflict(json.dumps(ex.args))

        except Exception as ex:  # pragma: no cover
            logger.error(ex)
            raise errors.HTTPServiceUnavailable

        else:

            if response:
                logger.info('block [{0}] deleted from vault {1}'
                            .format(block_id, vault_id))
                resp.status = falcon.HTTP_204

            else:
                logger.error('block [{0}] does not exist'.format(block_id))
                raise errors.HTTPNotFound

    def get_all(self, vault_id, file_id):

        vault = Vault.get(request.project_id, vault_id)

        assert vault is not None

        f = vault.get_file(file_id)

        if not f:
            abort(404)

        inmarker = int(request.params.get('marker', 0))
        limit = int(request.params.get('limit',
           conf.api_configuration.max_returned_num))

        # Get the block generator from the metadata driver.
        # Note: +1 on limit is to fetch one past the limt
        # for the purpose of determining if the
        # list was truncated
        retblks = deuce.metadata_driver.create_file_block_generator(
            request.project_id, vault_id, file_id, inmarker, limit + 1)

        resp = list(retblks)

        truncated = len(resp) > 0 and len(resp) == limit + 1
        outmarker = resp.pop()[1] if truncated else None

        if outmarker:
            query_args = {'marker': outmarker}
            query_args['limit'] = limit

            returl = set_qs(request.url, query_args)
            response.headers["X-Next-Batch"] = returl

        return resp

    def on_get(self, req, resp, vault_id):
        """List the blocks in the vault from storage-alone
        """
        vault = Vault.get(vault_id)
        if vault is None:
            logger.error("Vault [{0}] does not exist".format(vault_id))
            raise errors.HTTPNotFound

        inmarker = req.get_param("marker") if req.get_param("marker") else None
        limit = (
            req.get_param_as_int("limit")
            if req.get_param_as_int("limit")
            else conf.api_configuration.default_returned_num
        )

        # We actually fetch the user's requested
        # limit +1 to detect if the list is being
        # truncated or not.
        storage = BlockStorage.get(vault_id)
        storage_blocks = storage.get_blocks_generator(inmarker, limit + 1)

        responses = list(storage_blocks)

        # Was the list truncated? See note above about +1
        truncated = len(responses) > 0 and len(responses) == limit + 1

        outmarker = responses.pop().storage_block_id if truncated else None

        if outmarker:
            query_args = {"marker": outmarker}
            query_args["limit"] = limit
            returl = set_qs_on_url(req.url, query_args)
            resp.set_header("X-Next-Batch", returl)

        resp.body = json.dumps([response.storage_block_id for response in responses])

    def on_put(self, req, resp, vault_id, block_id):
        """Uploads a block into Deuce. The URL of the block
        is returned in the Location header
        """

        vault = Vault.get(vault_id)

        try:
            retval, storage_id = vault.put_block(
                block_id, req.stream.read(), req.content_length)
            resp.set_header('X-Storage-ID', str(storage_id))
            resp.set_header('X-Block-ID', str(block_id))

            block = vault.get_block(block_id)

            ref_cnt = 0
            ref_mod = 0

            if retval:
                ref_cnt = block.get_ref_count()
                ref_mod = block.get_ref_modified()

            resp.set_header('X-Block-Reference-Count', str(ref_cnt))
            resp.set_header('X-Ref-Modified', str(ref_mod))

            resp.status = (
                falcon.HTTP_201 if retval is True else falcon.HTTP_500)
            logger.info('block [{0}] added [{1}]'.format(block_id, storage_id))
        except ValueError as e:
            raise errors.HTTPPreconditionFailed('hash error')
        except BufferError as e:
            raise errors.HTTPPreconditionFailed(
                'content length did not match data length')

    def on_post(self, req, resp, vault_id, file_id):
        """This endpoint Assigns blocks to files
        """
        vault = Vault.get(vault_id)

        # caller tried to post to a vault that
        # does not exist
        if not vault:
            logger.error('Vault [{0}] does not exist'.format(vault_id))
            raise errors.HTTPBadRequestAPI('Vault does not exist')

        f = vault.get_file(file_id)

        if not f:
            logger.error('File [{0}] does not exist'.format(file_id))
            raise errors.HTTPNotFound

        if f.finalized:
            logger.error('Finalized file [{0}] '
                         'cannot be modified'.format(file_id))
            raise errors.HTTPConflict('Finalized file cannot be modified')

        body = req.stream.read(req.content_length)
        # TODO (TheSriram): Validate payload
        payload = json.loads(body.decode())
        block_ids, offsets = zip(*payload)

        missing_blocks = deuce.metadata_driver.has_blocks(vault_id, block_ids)
        deuce.metadata_driver.assign_blocks(vault_id, file_id, block_ids,
                                            offsets)

        resp.body = json.dumps(missing_blocks)

    def get_all(self, vault_id):

        vault = Vault.get(request.project_id, vault_id)

        if not vault:
            abort(404)

        inmarker = request.params.get('marker')
        limit = int(request.params.get('limit',
           conf.api_configuration.max_returned_num))

        # The +1 is to fetch one past the user's
        # requested limit so that we can determine
        # if the list was truncated or not
        files = vault.get_files(inmarker, limit + 1)

        resp = list(files)

        # Note: the list may not actually be truncated
        truncated = len(resp) == limit + 1

        outmarker = resp.pop().file_id if truncated else None

        if outmarker:
            query_args = {'marker': outmarker}
            query_args['limit'] = limit

            returl = set_qs(request.url, query_args)

            response.headers["X-Next-Batch"] = returl

        return resp

    def on_get(self, req, resp):

        # NOTE(TheSriram): get_param(param) automatically returns None
        # if param is not present
        inmarker = req.get_param('marker')
        limit = req.get_param_as_int('limit') if req.get_param_as_int('limit') else \
            conf.api_configuration.default_returned_num

        vaultlist = Vault.get_vaults_generator(
            inmarker, limit + 1)
        response = list(vaultlist)

        if not response:
            resp.body = json.dumps([])

        # Note: the list may not actually be truncated
        truncated = len(response) == limit + 1

        outmarker = response.pop() if truncated else None

        # Set x-next-batch resp header.
        if outmarker:
            query_args = {'marker': outmarker}
            query_args['limit'] = limit
            returl = set_qs_on_url(req.url, query_args)
            resp.set_header(name="X-Next-Batch", value=returl)

        # Set return json for vault URLs.
        p = urlparse(req.url)
        resp.body = json.dumps(dict(six.moves.map(lambda vaultname:
            (vaultname, {"url": p.scheme +
                '://' + p.netloc + p.path + '/' + vaultname}), response)))