Python FileStorage.FileStorage类代码示例

def do_full_backup(options):
    options.full = True
    dest = os.path.join(options.repository, gen_filename(options))
    if os.path.exists(dest):
        raise WouldOverwriteFiles('Cannot overwrite existing file: %s' % dest)
    # Find the file position of the last completed transaction.
    fs = FileStorage(options.file, read_only=True)
    # Note that the FileStorage ctor calls read_index() which scans the file
    # and returns "the position just after the last valid transaction record".
    # getSize() then returns this position, which is exactly what we want,
    # because we only want to copy stuff from the beginning of the file to the
    # last valid transaction record.
    pos = fs.getSize()
    # Save the storage index into the repository
    index_file = os.path.join(options.repository,
                              gen_filename(options, '.index'))
    log('writing index')
    fs._index.save(pos, index_file)
    fs.close()
    log('writing full backup: %s bytes to %s', pos, dest)
    sum = copyfile(options, dest, 0, pos)
    # Write the data file for this full backup
    datfile = os.path.splitext(dest)[0] + '.dat'
    fp = open(datfile, 'w')
    print >> fp, dest, 0, pos, sum
    fp.flush()
    os.fsync(fp.fileno())
    fp.close()
    if options.killold:
        delete_old_backups(options)

def analyze(path):
    fs = FileStorage(path, read_only=1)
    fsi = fs.iterator()
    report = Report()
    for txn in fsi:
        analyze_trans(report, txn)
    return report

class PackerTests(StorageTestBase):

    def setUp(self):
        self.started = 0

    def start(self):
        self.started =1
        self.path = tempfile.mktemp(suffix=".fs")
        self._storage = FileStorage(self.path)
        self.db = ZODB.DB(self._storage)
        self.do_updates()
        self.pid, self.exit = forker.start_zeo_server(self._storage, self.addr)

    def do_updates(self):
        for i in range(100):
            self._dostore()

    def tearDown(self):
        if not self.started:
            return
        self.db.close()
        self._storage.close()
        self.exit.close()
        try:
            os.kill(self.pid, 9)
        except os.error:
            pass
        try:
            os.waitpid(self.pid, 0)
        except os.error, err:
            ##print "waitpid failed", err
            pass
        removefs(self.path)

 def tearDown(self):
     self.storage.close()
     if self.recovered is not None:
         self.recovered.close()
     temp = FileStorage(self.dest)
     temp.close()
     ZODB.tests.util.TestCase.tearDown(self)

class DbAdapter:
    def __init__(self, path="data.db"):
        self.path = path

    def connect(self):
        self.storage = FileStorage(self.path)
        self.db = DB(self.storage)
        self.conn = self.db.open()
        return self.conn.root()

    def begin_transaction(self):
        transaction.begin()

    def commit(self):
        transaction.commit()

    def rollback(self):
        transaction.abort()

    def disconnect(self):
        self.conn.close()
        self.db.close()
        self.storage.close()
        if os.path.exists(self.path + ".lock"):
            os.remove(self.path + ".lock")

    def testBadTransaction(self):
        # Find transaction headers and blast them.

        L = self.storage.undoLog()
        r = L[3]
        tid = base64.decodestring(r["id"] + "\n")
        pos1 = self.storage._txn_find(tid, 0)

        r = L[8]
        tid = base64.decodestring(r["id"] + "\n")
        pos2 = self.storage._txn_find(tid, 0)

        self.storage.close()

        # Overwrite the entire header.
        f = open(self.path, "a+b")
        f.seek(pos1 - 50)
        f.write("\0" * 100)
        f.close()
        output = self.recover()
        self.assert_('error' in output, output)
        self.recovered = FileStorage(self.dest)
        self.recovered.close()
        os.remove(self.path)
        os.rename(self.dest, self.path)

        # Overwrite part of the header.
        f = open(self.path, "a+b")
        f.seek(pos2 + 10)
        f.write("\0" * 100)
        f.close()
        output = self.recover()
        self.assert_('error' in output, output)
        self.recovered = FileStorage(self.dest)
        self.recovered.close()

def do_incremental_backup(options, reposz, repofiles):
    options.full = False
    dest = os.path.join(options.repository, gen_filename(options))
    if os.path.exists(dest):
        raise WouldOverwriteFiles('Cannot overwrite existing file: %s' % dest)
    # Find the file position of the last completed transaction.
    fs = FileStorage(options.file, read_only=True)
    # Note that the FileStorage ctor calls read_index() which scans the file
    # and returns "the position just after the last valid transaction record".
    # getSize() then returns this position, which is exactly what we want,
    # because we only want to copy stuff from the beginning of the file to the
    # last valid transaction record.
    pos = fs.getSize()
    log('writing index')
    index_file = os.path.join(options.repository,
                              gen_filename(options, '.index'))
    fs._index.save(pos, index_file)
    fs.close()
    log('writing incremental: %s bytes to %s',  pos-reposz, dest)
    sum = copyfile(options, dest, reposz, pos - reposz)
    # The first file in repofiles points to the last full backup.  Use this to
    # get the .dat file and append the information for this incrementatl to
    # that file.
    fullfile = repofiles[0]
    datfile = os.path.splitext(fullfile)[0] + '.dat'
    # This .dat file better exist.  Let the exception percolate if not.
    fp = open(datfile, 'a')
    print >> fp, dest, reposz, pos, sum
    fp.flush()
    os.fsync(fp.fileno())
    fp.close()

class HistoryFreeFromFileStorage(
        RelStorageTestBase,
        UndoableRecoveryStorage,
        ):

    keep_history = False

    def setUp(self):
        self.open(create=1)
        self._storage.zap_all()
        self._dst = self._storage
        self._storage = FileStorage("Source.fs", create=True)

    def tearDown(self):
        self._storage.close()
        self._dst.close()
        self._storage.cleanup()
        self._dst.cleanup()

    def new_dest(self):
        return self._dst

    def compare(self, src, dest):
        # The dest storage has a truncated copy of dest, so
        # use compare_truncated() instead of compare_exact().
        self.compare_truncated(src, dest)

class Base(object):

    def __init__(self, path, authkey):
        if not os.path.exists(path):
            os.makedirs(path)
        self._path = path
        self.authkey = authkey

        path = os.path.join(path, 'graph.fs')
        self.storage = FileStorage(path)
        self.db = DB(self.storage)

    def path(self):
        return self._path

    def process(self, connection):
        (func, args) = connection
        self.connection = func(*args)

    def recv(self):
        return self.connection.recv()

    def send(self, message):
        self.connection.send(message)
        self.connection.close()

    def open(self):
        return self.db.open()

    def close(self):
        transaction.get().abort()
        self.db.close()
        self.storage.close()

 def tearDown(self):
     fsetup = functional.FunctionalTestSetup()
     # close the filestorage files now by calling the original
     # close on our storage instance
     FileStorage.close(fsetup.base_storage)
     fsetup.base_storage = self.original
     fsetup.tearDown()
     fsetup.tearDownCompletely()

 def tearDown(self):
     self.storage.close()
     if self.recovered is not None:
         self.recovered.close()
     self.storage.cleanup()
     temp = FileStorage(self.dest)
     temp.close()
     temp.cleanup()

def main(path=None):
    verbose = 0
    if path is None:
        import sys
        import getopt

        opts, args = getopt.getopt(sys.argv[1:], "v")
        for k, v in opts:
            if k == "-v":
                verbose += 1

        path, = args


    fs = FileStorage(path, read_only=1)

    # Set of oids in the index that failed to load due to POSKeyError.
    # This is what happens if undo is applied to the transaction creating
    # the object (the oid is still in the index, but its current data
    # record has a backpointer of 0, and POSKeyError is raised then
    # because of that backpointer).
    undone = {}

    # Set of oids that were present in the index but failed to load.
    # This does not include oids in undone.
    noload = {}

    for oid in fs._index.keys():
        try:
            data, serial = fs.load(oid, "")
        except (KeyboardInterrupt, SystemExit):
            raise
        except POSKeyError:
            undone[oid] = 1
        except:
            if verbose:
                traceback.print_exc()
            noload[oid] = 1

    inactive = noload.copy()
    inactive.update(undone)
    for oid in fs._index.keys():
        if oid in inactive:
            continue
        data, serial = fs.load(oid, "")
        refs = get_refs(data)
        missing = [] # contains 3-tuples of oid, klass-metadata, reason
        for ref, klass in refs:
            if klass is None:
                klass = '<unknown>'
            if ref not in fs._index:
                missing.append((ref, klass, "missing"))
            if ref in noload:
                missing.append((ref, klass, "failed to load"))
            if ref in undone:
                missing.append((ref, klass, "object creation was undone"))
        if missing:
            report(oid, data, serial, missing)

def analyze(path, use_dbm):
    fs = FileStorage(path, read_only=1)
    fsi = fs.iterator()
    report = Report(use_dbm)
    for txn in fsi:
        analyze_trans(report, txn)
    if use_dbm:
        shutil.rmtree(report.temp_dir)
    return report

    def checkBackwardTimeTravelWithRevertWhenStale(self):
        # If revert_when_stale is true, when the database
        # connection is stale (such as through failover to an
        # asynchronous slave that is not fully up to date), the poller
        # should notice that backward time travel has occurred and
        # invalidate all objects that have changed in the interval.
        self._storage = self.make_storage(revert_when_stale=True)

        import os
        import shutil
        import tempfile
        from ZODB.FileStorage import FileStorage

        db = DB(self._storage)
        try:
            transaction.begin()
            c = db.open()
            r = c.root()
            r["alpha"] = PersistentMapping()
            transaction.commit()

            # To simulate failover to an out of date async slave, take
            # a snapshot of the database at this point, change some
            # object, then restore the database to its earlier state.

            d = tempfile.mkdtemp()
            try:
                transaction.begin()
                fs = FileStorage(os.path.join(d, "Data.fs"))
                fs.copyTransactionsFrom(c._storage)

                r["beta"] = PersistentMapping()
                transaction.commit()
                self.assertTrue("beta" in r)

                c._storage.zap_all(reset_oid=False, slow=True)
                c._storage.copyTransactionsFrom(fs)

                fs.close()
            finally:
                shutil.rmtree(d)

            # r should still be in the cache.
            self.assertTrue("beta" in r)

            # Now sync, which will call poll_invalidations().
            c.sync()

            # r should have been invalidated
            self.assertEqual(r._p_changed, None)

            # r should be reverted to its earlier state.
            self.assertFalse("beta" in r)

        finally:
            db.close()

 def tearDown(self):
     fsetup = functional.FunctionalTestSetup(self.config_file)
     # close the filestorage files now by calling the original
     # close on our storage instance
     FileStorage.close(fsetup.base_storage)
     # replace the storage with the original, so functionalsetup
     # can do what it wants with it
     fsetup.base_storage = self.original
     fsetup.tearDown()
     fsetup.tearDownCompletely()