Python DB.execute方法代码示例

# 需要导入模块: from lib.db import DB [as 别名]
# 或者: from lib.db.DB import execute [as 别名]

#.........这里部分代码省略.........
        if self.options.store:
            store1 = self.config.storage[self.options.store].copy()
            store2 = store1
        else:
            #    Make the store about 3x the options size
            s, dummy, dummy = utils.from_readable_form(self.options.size)
            store_size = utils.readable_form(s * 3)
            store1 = FolderStore("teststore1", store_size, True, os.path.join(self.store_folder, "teststore1"))
            store2 = FolderStore("teststore2", store_size, True, os.path.join(self.store_folder, "teststore2"))
            self.config.storage[store1.name] = store1
            self.config.storage[store2.name] = store2

        backup1 = Backup("testbackup1")
        backup1.include_folders = [self.files_folder]
        backup1.include_packages = True
        backup1.exclude_types = ["Music"]
        backup1.exclude_patterns = []
        backup1.store = store1.name
        backup1.notify_msg = False
        self.config.backups[backup1.name] = backup1

        backup2 = Backup("testbackup2")
        backup2.include_folders = [self.files_folder]
        backup2.include_packages = True
        backup2.exclude_patterns = []
        backup1.exclude_types = ["Videos", "Programs"]
        backup2.store = store2.name
        backup2.notify_msg = False
        self.config.backups[backup2.name] = backup2

    def fill_files(self, remaining=None, root=None):
        log.trace("fill_files")
        if not remaining:
            #    First time in...
            remaining, dummy, dummy = utils.from_readable_form(self.options.size)
            root = self.files_folder
            
        list = [root]
        done = False
        while not done:
            newlist = []
            for folder in list:
                for dname in ["dir1", "dir2", "dir3"]:
                    path = os.path.join(folder, dname)
                    utils.makedirs(path)
                    newlist.append(path)
                for fname in ["f1.avi", "f2.mp3", "f3.exe", "f4.txt"]:
                    path = os.path.join(folder, fname)
                    with open(path, "w") as f:
                        f.write(self.teststring1)
                    remaining -= len(self.teststring1)
                if remaining < 0:
                    done = True
                    break
            list = newlist
            
        return
                
            


    def make_folders(self):
        log.trace("make_folders")
        if not os.path.isdir(self.test_folder):
            os.makedirs(self.test_folder)

#        if not os.path.isdir(self.store_folder):
#            os.makedirs(self.store_folder)
        if not os.path.isdir(self.files_folder):
            os.makedirs(self.files_folder)
        if not os.path.isdir(self.restore_folder):
            os.makedirs(self.restore_folder)


    def cleanup(self):
        log.trace("Cleanup")
        self.db.execute("delete from messages where message_id > ?", (self.max_message_id,))
        self.db.execute("delete from versions where version_id > ?", (self.max_version_id,))
        self.db.execute("delete from fs where fs_id > ?", (self.max_fs_id,))
        self.db.execute("delete from runs where run_id > ?", (self.max_run_id,))


        if self.options.store:
            stores = self.options.store
        else:
            stores = ["teststore1", "teststore2"]
        for name in stores:
            log.info("Cleaning up", name)
            store = self.config.storage[name].copy()
            store.connect()
            store.delete_store_data()
            store.disconnect()

        if not self.options.store:
            del self.config.storage["teststore1"]
            del self.config.storage["teststore2"]
        del self.config.backups["testbackup1"]
        del self.config.backups["testbackup2"]

        shutil.rmtree(self.test_folder)

# 需要导入模块: from lib.db import DB [as 别名]
# 或者: from lib.db.DB import execute [as 别名]

#.........这里部分代码省略.........
        self.config.backups[self.backup.name] = self.backup

        #    build an options object for use with the backup
        self.options = BlankClass()
        self.options.dry_run = False
        self.options.message = False
        self.options.email = False
        self.options.shutdown = False
        self.options.norecurse = False
        
        #    How many cycles?
        self.cycles = 20

    def tearDown(self):
        self.config.data_passphrase = self.old_pass
        #    Remove all DB records created during this test
        self.clean_db()
        shutil.rmtree(self.test_folder)
        self.assertFalse(os.path.isdir(self.test_folder))


    def testLongRun(self):

        #    Run a full backup
        b = Run(self.backup.name, const.FullBackup, self.options)
        b.run()

        for cycle in xrange(self.cycles):
            print(str(cycle)+"\r")
            time.sleep(1)
            
            #    Change some files
            with open(os.path.join(self.files_folder, "incr"), "w") as f:
                f.write(os.urandom(100))

            with open(os.path.join(self.files_folder, str(cycle)), "w") as f:
                f.write(os.urandom(100))
            

            #    Run an incr backup
            b = Run(self.backup.name, const.IncrBackup, self.options)
            b.run()

        
        #    Attempt to restore every file
        r = Restore(self.restore_folder, [self.files_folder],
                                          datetime.now(), self.options)
        r.run()
        
        #    Lets break it
#        os.remove(os.path.join(self.restore_folder, self.files_folder[1:], "1"))
#        with open(os.path.join(self.files_folder, "incr"), "w") as f:
#            f.write("-1")       
#        with open(os.path.join(self.restore_folder, self.files_folder[1:], "8"), "w") as f:
#            f.write("-1")       

        #    Check that the restored folder and original folder are identical
        left = unicode(self.files_folder)
        right = unicode(os.path.join(self.restore_folder, self.files_folder[1:]))
        d = utils.dircmp(left, right)

        self.assertEqual(d.left_only, set())
        self.assertEqual(d.right_only, set())
        self.assertEqual(d.diff_files, set())
        self.assertTrue(len(d.same_files) > 0)

        #    Check that all files are in the DB
        for folder, _, local_files in os.walk(self.files_folder):
            for file in local_files:
                path = os.path.join(file, folder)
                #    This will raise an exception if it does not exist
                self.db.select_path(path, build=False)



############################################################################
#
#    Utility Routines
#
############################################################################

    def mark_db_ids(self):
        self.max_fs_id = self.db.query("select max(fs_id) from fs", ())[0][0]
        if self.max_fs_id is None:
            self.max_fs_id = 0
        self.max_version_id = self.db.query("select max(version_id) from versions", ())[0][0]
        if self.max_version_id is None:
            self.max_version_id = 0
        self.max_run_id = self.db.query("select max(run_id) from runs", ())[0][0]
        if self.max_run_id is None:
            self.max_run_id = 0
        self.max_message_id = self.db.query("select max(message_id) from messages", ())[0][0]
        if self.max_message_id is None:
            self.max_message_id = 0

    def clean_db(self):
        self.db.execute("delete from messages where message_id > ?", (self.max_message_id,))
        self.db.execute("delete from versions where version_id > ?", (self.max_version_id,))
        self.db.execute("delete from fs where fs_id > ?", (self.max_fs_id,))
        self.db.execute("delete from runs where run_id > ?", (self.max_run_id,))

# 需要导入模块: from lib.db import DB [as 别名]
# 或者: from lib.db.DB import execute [as 别名]

#.........这里部分代码省略.........
        run = runs[0]


        v = Verify("testbackup", run.start_time)
        self.assertTrue(v.run())


    def testBadVerify(self):
        #    Run a full backup
        b = Run("testbackup", const.FullBackup, self.options)
        b.run()

        #    Get the times
        runs = self.db.runs("testbackup")
        run = runs[0]

        #    Get the location of the data file from the streamer
        streamer = StreamOut(None, self.store, b.backup_folder)
        datafile = os.path.join(self.store.root, streamer.get_path(0))
        size = os.path.getsize(datafile)
        #    Now corrupt the data file a little
        with open(datafile, "r+b") as f:
            f.seek(size // 2, 0)
            f.write("X")


        v = Verify("testbackup", run.start_time)
        self.assertRaises(Exception, v.run)


    def testBadConfig(self):
        #    Run a full backup
        b = Run("testbackup", const.FullBackup, self.options)
        b.run()

        #    Get the times
        runs = self.db.runs("testbackup")
        run = runs[0]

        #    Delete The Config File
        configfile = os.path.join(run.folder, const.ConfigName)
        self.store.remove_file(configfile)


        v = Verify("testbackup", run.start_time)
        self.assertRaises(Exception, v.run)

    def testBadVerifyEncrypted(self):
        backup = self.config.backups[self.backup.name]
        backup.encrypt = True
        self.config.backups[backup.name] = backup

        #    Run a full backup
        b = Run("testbackup", const.FullBackup, self.options)
        b.run()

        #    Get the times
        runs = self.db.runs("testbackup")
        run = runs[0]

        #    Get the location of the data file from the streamer
        streamer = StreamOut(None, self.store, b.backup_folder)
        datafile = os.path.join(self.store.root, streamer.get_path(0))
        #    Now corrupt the data file a little
        size = os.path.getsize(datafile)
        with open(datafile, "r+b") as f:
            f.seek(size // 2, 0)
            f.write("X")


        v = Verify("testbackup", run.start_time)
        self.assertRaises(Exception, v.run)



############################################################################
#
#    Utility Routines
#
############################################################################

    def mark_db_ids(self):
        self.max_fs_id = self.db.query("select max(fs_id) from fs", ())[0][0]
        if self.max_fs_id is None:
            self.max_fs_id = 0
        self.max_version_id = self.db.query("select max(version_id) from versions", ())[0][0]
        if self.max_version_id is None:
            self.max_version_id = 0
        self.max_run_id = self.db.query("select max(run_id) from runs", ())[0][0]
        if self.max_run_id is None:
            self.max_run_id = 0
        self.max_message_id = self.db.query("select max(message_id) from messages", ())[0][0]
        if self.max_message_id is None:
            self.max_message_id = 0

    def clean_db(self):
        self.db.execute("delete from messages where message_id > ?", (self.max_message_id,))
        self.db.execute("delete from versions where version_id > ?", (self.max_version_id,))
        self.db.execute("delete from fs where fs_id > ?", (self.max_fs_id,))
        self.db.execute("delete from runs where run_id > ?", (self.max_run_id,))

# 需要导入模块: from lib.db import DB [as 别名]
# 或者: from lib.db.DB import execute [as 别名]

#.........这里部分代码省略.........

        for t, exists, contents in [(t0, False, None), (t1, True, "1"), (t2, True, "2"), (None, True, "2")]:
            #    Attempt to restore most recent of ALL files
            #    This tests the default restore.
            r = Restore(self.restore_folder, [self.files_folder], t, self.options)
            r.run()
            if exists:
                with open(restore_file, "r") as f:
                    self.assertEqual(f.read(), contents)
            else:
                self.assertFalse(os.path.exists(restore_file))
            #    clean
            shutil.rmtree(self.restore_folder)
            utils.makedirs(self.restore_folder)


    def test7bitFilenames(self):
        #    Make some 7 bit filenames
        strange_folder = os.path.join(self.files_folder, "strange")
        utils.makedirs(strange_folder)
        for i in xrange(1, 117, 10):
            name = "".join([chr(j) for j in xrange(i, i + 10) if chr(j) != "/"])
            path = os.path.join(strange_folder, name)
            with open(path, "w") as f:
                f.write(os.urandom(100))

        self.backup_restore_compare()

    def testUnicodeFilenames(self):
        #    Make some unicode bit filenames
        #    Clean out the ordinary files
        shutil.rmtree(self.files_folder)
        utils.makedirs(self.files_folder)
        unicode_folder = os.path.join(unicode(self.files_folder), u"unicode")
        utils.makedirs(unicode_folder)
        for i in xrange(1000, 1200, 10):
            name = u"".join([unichr(j) for j in xrange(i, i + 10) if unichr(j) != u"/"])
            path = os.path.join(unicode_folder, name)
            with open(path, "w") as f:
                f.write(os.urandom(10))

        self.backup_restore_compare()

    def backup_restore_compare(self):
        #    Run a full backup
        b = Run("testbackup", const.FullBackup, self.options)
        b.run()

        #    Make sure we have ticked to another second since the start of the last backup.
        while datetime.now() - b.start_time < timedelta(seconds=1):
            time.sleep(0.01)


        #    Attempt to restore every file
        r = Restore(self.restore_folder, [self.files_folder],
                                          datetime.now(), self.options)
        r.run()

        #    Check that the restored folder and original folder are identical
        left = unicode(self.files_folder)
        right = unicode(os.path.join(self.restore_folder, self.files_folder[1:]))
        d = utils.dircmp(left, right)

        self.assertEqual(d.left_only, set())
        self.assertEqual(d.right_only, set())
        self.assertEqual(d.diff_files, set())
        self.assertTrue(len(d.same_files) > 0)

        #    Check that all files are in the DB
        for folder, _, local_files in os.walk(self.files_folder):
            for file in local_files:
                path = os.path.join(file, folder)
                #    This will raise an exception if it does not exist
                self.db.select_path(path, build=False)

############################################################################
#
#    Utility Routines
#
############################################################################

    def mark_db_ids(self):
        self.max_fs_id = self.db.query("select max(fs_id) from fs", ())[0][0]
        if self.max_fs_id is None:
            self.max_fs_id = 0
        self.max_version_id = self.db.query("select max(version_id) from versions", ())[0][0]
        if self.max_version_id is None:
            self.max_version_id = 0
        self.max_run_id = self.db.query("select max(run_id) from runs", ())[0][0]
        if self.max_run_id is None:
            self.max_run_id = 0
        self.max_message_id = self.db.query("select max(message_id) from messages", ())[0][0]
        if self.max_message_id is None:
            self.max_message_id = 0

    def clean_db(self):
        self.db.execute("delete from messages where message_id > ?", (self.max_message_id,))
        self.db.execute("delete from versions where version_id > ?", (self.max_version_id,))
        self.db.execute("delete from fs where fs_id > ?", (self.max_fs_id,))
        self.db.execute("delete from runs where run_id > ?", (self.max_run_id,))

# 需要导入模块: from lib.db import DB [as 别名]
# 或者: from lib.db.DB import execute [as 别名]
 def update_past_issues_processed(self):
   """Update processed issues with current batch name and list of issues processed"""
   sql = "INSERT INTO issues_processed (`issue_key`, `batch_name`) VALUES (%s, %s)"
   for issue in self.issue_queue:
     DB.execute(sql, (issue.key(), self.full_name))

# 需要导入模块: from lib.db import DB [as 别名]
# 或者: from lib.db.DB import execute [as 别名]

#.........这里部分代码省略.........
            self.orig_type = self.type
            self.check_old_backups()
            self.do_backup()
        finally:
            self.lock.release()

    def check_old_backups(self):
        '''
        We have got the lock, but if there was a crash, there may be a "running"
        state backup left behind. Note that we *know* its not running because
        the lock is gone.
        
        Check for it and fail it if there is.
        '''
        log.debug("Checking for dead backups")
        runs = self.db.runs(self.backup.name)
        runs = [run for run in runs if run.status == const.StatusRunning]

        #    It looks like there is a run that is still running.
        for run in runs:
            log.warn("A prior run crashed. Cleaning up %s/%s" % (run.name, run.start_time_str))
            #    Get the store
            log.debug("Attempting to delete remote run data")
            try:
                self.store.delete_run_data(run)
            except:
                pass
            #    Delete the entries in the database (but not the failed run itself)
            #    This means the messages will persist, so we can see the usage.
            log.debug("Attempting to delete DB run data")
            self.db.delete_run_versions(self.run_id)
            #    Update the status
            log.debug("Setting status to failed and logging")
            self.db.execute("update runs set status = ? where run_id = ?", (const.StatusFailed, run.run_id))
            self.db.execute("insert into messages (run_id, message, time) values (?, ?, ?)",
                            (run.run_id, _("Backup run crashed. It was cleaned up."),
                             datetime.now().strftime(const.DateTimeFormat)))


        #        If there are *no* full backups in the history, then this run MUST be a full,
        #        even if an incremental is requested. Actually a request for incremental will
        #        grab all files anyway, but still... lets make the name match the contents.
        runs = self.db.runs(self.backup.name)
        full_count = len([run for run in runs
                            if run.type == const.FullBackup and
                            run.status == const.StatusSuccess])
        log.debug("Full backups: %d" % full_count)
        if full_count == 0 and self.type != const.FullBackup:
            log.debug("Resetting type to Full")
            self.type = const.FullBackup


    def do_backup(self)      :
        self.start_time = datetime.now()
        self.nfiles = 0
        self.nfolders = 0
        self.bytes = 0
        success = False
        message = ""

        self.backup_folder = os.path.join(self.backup.name, self.start_time.strftime(const.DateTimeFormat) + " " + self.type)
        if not self.dry_run:
            self.run_id = self.db.start_run(self.backup.name, self.backup.store, self.type, self.start_time)
            msg = _("Backup {server}/{backup}/{type} beginning").format(
                                                            server=utils.get_hostname(), 
                                                            backup=self.backup.name,