Python SQLAlchemyAll.FSObject类代码示例

    def do_import(self):
        """Take care of creating the database structure, delegating
        the loading of the contest data and putting them on the
        database.

        """
        logger.info("Creating database structure.")
        if self.drop:
            try:
                with SessionGen() as session:
                    FSObject.delete_all(session)
                    session.commit()
                metadata.drop_all()
            except sqlalchemy.exc.OperationalError as error:
                logger.critical("Unable to access DB.\n%r" % error)
                return False
        try:
            metadata.create_all()
        except sqlalchemy.exc.OperationalError as error:
            logger.critical("Unable to access DB.\n%r" % error)
            return False

        contest = Contest.import_from_dict(self.loader.import_contest(self.path))

        logger.info("Creating contest on the database.")
        with SessionGen() as session:
            session.add(contest)
            logger.info("Analyzing database.")
            session.commit()
            contest_id = contest.id
            analyze_all_tables(session)

        logger.info("Import finished (new contest id: %s)." % contest_id)

        return True

    def put_file(self, digest, origin, description=""):
        """See FileCacherBackend.put_file().

        """
        with SessionGen() as session:

            # Check digest uniqueness
            if FSObject.get_from_digest(digest, session) is not None:
                logger.debug("File %s already on database, "
                             "dropping this one." % digest)
                session.rollback()

            # If it is not already present, copy the file into the
            # lobject
            else:
                fso = FSObject(description=description)
                logger.debug("Sending file %s to the database." % digest)
                with open(origin, 'rb') as temp_file:
                    with fso.get_lobject(session, mode='wb') as lobject:
                        logger.debug("Large object created.")
                        buf = temp_file.read(self.CHUNK_SIZE)
                        while buf != '':
                            while len(buf) > 0:
                                written = lobject.write(buf)
                                buf = buf[written:]
                                if self.service is not None:
                                    self.service._step()
                            buf = temp_file.read(self.CHUNK_SIZE)
                fso.digest = digest
                session.add(fso)
                session.commit()
                logger.debug("File %s sent to the database." % digest)

 def _prepare_db(self):
     logger.info("Creating database structure.")
     if self.drop:
         try:
             with SessionGen() as session:
                 FSObject.delete_all(session)
                 session.commit()
             metadata.drop_all()
         except sqlalchemy.exc.OperationalError as error:
             logger.critical("Unable to access DB.\n%r" % error)
             return False
     try:
         metadata.create_all()
     except sqlalchemy.exc.OperationalError as error:
         logger.critical("Unable to access DB.\n%r" % error)
         return False

    def delete(self, digest):
        """See FileCacherBackend.delete().

        """
        with SessionGen() as session:
            fso = FSObject.get_from_digest(digest, session)
            fso.delete()
            session.commit()

    def describe(self, digest):
        """See FileCacherBackend.describe().

        """
        with SessionGen() as session:
            fso = FSObject.get_from_digest(digest, session)
            if fso is not None:
                return fso.description
            else:
                return None

    def get_size(self, digest):
        """See FileCacherBackend.get_size().

        """
        with SessionGen() as session:
            fso = FSObject.get_from_digest(digest, session)
            if fso is not None:
                with fso.get_lobject(session, mode='rb') as lobject:
                    return lobject.seek(0, os.SEEK_END)
            else:
                return None

    def delete(self, digest):
        """Delete from cache and FS the file with that digest.

        digest (string): the file to delete.

        """
        self.delete_from_cache(digest)
        with SessionGen() as session:
            fso = FSObject.get_from_digest(digest, session)
            fso.delete()
            session.commit()

    def put_file(self, digest, origin, description=""):
        """See FileCacherBackend.put_file().

        """
        try:
            with SessionGen() as session:

                # Check digest uniqueness
                if FSObject.get_from_digest(digest, session) is not None:
                    logger.debug("File %s already on database, "
                                 "dropping this one." % digest)
                    session.rollback()

                # If it is not already present, copy the file into the
                # lobject
                else:
                    fso = FSObject(description=description)
                    logger.debug("Sending file %s to the database." % digest)
                    with open(origin, 'rb') as temp_file:

                        with fso.get_lobject(session, mode='wb') \
                                as lobject:
                            logger.debug("Large object created.")
                            buf = temp_file.read(self.CHUNK_SIZE)
                            while buf != '':
                                while len(buf) > 0:
                                    written = lobject.write(buf)
                                    buf = buf[written:]
                                    # Cooperative yield
                                    gevent.sleep(0)
                                buf = temp_file.read(self.CHUNK_SIZE)

                    fso.digest = digest
                    session.add(fso)
                    session.commit()
                    logger.debug("File %s sent to the database." % digest)

        except IntegrityError:
            logger.warning("File %s caused an IntegrityError, ignoring..."
                           % digest)

    def describe(digest):
        """Return the description of a file given its digest.

        digest (string): the digest to describe.
        return (string): the description associated.

        """
        with SessionGen() as session:
            fso = FSObject.get_from_digest(digest, session)
            if fso is not None:
                return fso.description
            else:
                return None

    def get_file(self, digest, dest):
        """See FileCacherBackend.get_file().

        """
        with open(dest, 'wb') as temp_file:
            # hasher = hashlib.sha1()
            with SessionGen() as session:
                fso = FSObject.get_from_digest(digest, session)

                # Copy the file into the lobject
                with fso.get_lobject(mode='rb') as lobject:
                    buf = lobject.read(self.CHUNK_SIZE)
                    while buf != '':
                        # hasher.update(buf)
                        temp_file.write(buf)
                        if self.service is not None:
                            self.service._step()
                        buf = lobject.read(self.CHUNK_SIZE)

    def get_file(self, digest, path=None, file_obj=None,
                 string=False, temp_path=False, temp_file_obj=False):
        """Get a file from the storage, possibly using the cache if
        the file is available there.

        digest (string): the sha1 sum of the file.
        path (string): a path where to save the file.
        file_obj (file): a handler where to save the file (that is not
                         closed at return).
        string (bool): True to return content as a string.
        temp_path (bool): True to return path of a temporary file with
                          that content. The file is reserved to the
                          caller, who has the duty to unlink it.
        temp_file-obj (bool): True to return a file object opened to a
                              temporary file with that content. The
                              file is reserved to the caller. Use this
                              method only for debugging purpose, as it
                              leave a file lying in the temporary
                              directory of FileCacher.

        """
        if [string, temp_path, temp_file_obj].count(True) > 1:
            raise ValueError("Ask for at most one amongst content, "
                             "temp path and temp file obj.")

        cache_path = os.path.join(self.obj_dir, digest)
        cache_exists = os.path.exists(cache_path)

        logger.debug("Getting file %s" % (digest))

        if not cache_exists:
            logger.debug("File %s not in cache, downloading "
                         "from database." % digest)

            temp_file, temp_filename = tempfile.mkstemp(dir=self.tmp_dir)
            temp_file = os.fdopen(temp_file, "wb")

            # Receives the file from the database
            with open(temp_filename, 'wb') as temp_file:
                # hasher = hashlib.sha1()
                with SessionGen() as session:
                    fso = FSObject.get_from_digest(digest, session)

                    # Copy the file into the lobject
                    with fso.get_lobject(mode='rb') as lobject:
                        buf = lobject.read(self.CHUNK_SIZE)
                        while buf != '':
                            # hasher.update(buf)
                            temp_file.write(buf)
                            if self.service is not None:
                                self.service._step()
                            buf = lobject.read(self.CHUNK_SIZE)

            # And move it in the cache
            shutil.move(temp_filename, cache_path)

            logger.debug("File %s downloaded." % digest)

        # Saving to path
        if path is not None:
            shutil.copy(cache_path, path)

        # Saving to file object
        if file_obj is not None:
            with open(cache_path, "rb") as file_:
                shutil.copyfileobj(file_, file_obj)

        # Returning string?
        if string:
            with open(cache_path, "rb") as cache_file:
                return cache_file.read()

        # Returning temporary file?
        elif temp_path:
            temp_file, temp_filename = tempfile.mkstemp(dir=self.tmp_dir)
            os.close(temp_file)
            shutil.copy(cache_path, temp_filename)
            return temp_filename

        # Returning temporary file object?
        elif temp_file_obj:
            temp_file, temp_filename = tempfile.mkstemp(dir=self.tmp_dir)
            os.close(temp_file)
            shutil.copy(cache_path, temp_filename)
            temp_file = open(temp_filename, "rb")
            return temp_file

    def put_file(self, description="", binary_data=None,
                 file_obj=None, path=None):
        """Put a file in the storage, and keep a copy locally. The
        caller has to provide exactly one among binary_data, file_obj
        and path.

        description (string): a human-readable description of the
                              content.
        binary_data (string): the content of the file to send.
        file_obj (file): the file-like object to send.
        path (string): the file to send.

        """
        temp_fd, temp_path = tempfile.mkstemp(dir=self.tmp_dir)
        os.close(temp_fd)

        # Input checking
        if [binary_data, file_obj, path].count(None) != 2:
            error_string = "No content (or too many) specified in put_file."
            logger.error(error_string)
            raise ValueError(error_string)

        logger.debug("Reading input file to store on the database.")

        # Copy the file content, whatever forms it arrives, into the
        # temporary file
        # TODO - This could be long lasting: probably it would be wise
        # to call self.service._step() periodically, but this would
        # require reimplementing of shutil functions
        if path is not None:
            shutil.copy(path, temp_path)
        elif binary_data is not None:
            with open(temp_path, 'wb') as temp_file:
                temp_file.write(binary_data)
        else:  # file_obj is not None.
            with open(temp_path, 'wb') as temp_file:
                shutil.copyfileobj(file_obj, temp_file)

        hasher = hashlib.sha1()
        fso = FSObject(description=description)

        # Calculate the file SHA1 digest
        with open(temp_path, 'rb') as temp_file:
            buf = temp_file.read(self.CHUNK_SIZE)
            while buf != '':
                hasher.update(buf)
                buf = temp_file.read(self.CHUNK_SIZE)
        digest = hasher.hexdigest()

        logger.debug("File has digest %s." % digest)

        # Check the digest uniqueness
        with SessionGen() as session:
            if FSObject.get_from_digest(digest, session) is not None:
                logger.debug("File %s already on database, "
                             "dropping this one." % digest)
                session.rollback()

            # If it is not already present, copy the file into the
            # lobject
            else:
                logger.debug("Sending file %s to the database." % digest)
                with open(temp_path, 'rb') as temp_file:
                    with fso.get_lobject(session, mode='wb') as lobject:
                        logger.debug("Large object created.")
                        buf = temp_file.read(self.CHUNK_SIZE)
                        while buf != '':
                            while len(buf) > 0:
                                written = lobject.write(buf)
                                buf = buf[written:]
                                if self.service is not None:
                                    self.service._step()
                            buf = temp_file.read(self.CHUNK_SIZE)
                fso.digest = digest
                session.add(fso)
                session.commit()
                logger.debug("File %s sent to the database." % digest)

        # Move the temporary file in the cache
        shutil.move(temp_path,
                    os.path.join(self.obj_dir, digest))

        return digest