Python FileStorage.load方法代码示例

# 需要导入模块: from ZODB.FileStorage import FileStorage [as 别名]
# 或者: from ZODB.FileStorage.FileStorage import load [as 别名]
def main(path=None):
    verbose = 0
    if path is None:
        import sys
        import getopt

        opts, args = getopt.getopt(sys.argv[1:], "v")
        for k, v in opts:
            if k == "-v":
                verbose += 1

        path, = args


    fs = FileStorage(path, read_only=1)

    # Set of oids in the index that failed to load due to POSKeyError.
    # This is what happens if undo is applied to the transaction creating
    # the object (the oid is still in the index, but its current data
    # record has a backpointer of 0, and POSKeyError is raised then
    # because of that backpointer).
    undone = {}

    # Set of oids that were present in the index but failed to load.
    # This does not include oids in undone.
    noload = {}

    for oid in fs._index.keys():
        try:
            data, serial = fs.load(oid, "")
        except (KeyboardInterrupt, SystemExit):
            raise
        except POSKeyError:
            undone[oid] = 1
        except:
            if verbose:
                traceback.print_exc()
            noload[oid] = 1

    inactive = noload.copy()
    inactive.update(undone)
    for oid in fs._index.keys():
        if oid in inactive:
            continue
        data, serial = fs.load(oid, "")
        refs = get_refs(data)
        missing = [] # contains 3-tuples of oid, klass-metadata, reason
        for ref, klass in refs:
            if klass is None:
                klass = '<unknown>'
            if ref not in fs._index:
                missing.append((ref, klass, "missing"))
            if ref in noload:
                missing.append((ref, klass, "failed to load"))
            if ref in undone:
                missing.append((ref, klass, "object creation was undone"))
        if missing:
            report(oid, data, serial, missing)

# 需要导入模块: from ZODB.FileStorage import FileStorage [as 别名]
# 或者: from ZODB.FileStorage.FileStorage import load [as 别名]
def main(path, search_oids):
    fs = FileStorage(path, read_only=1)

    # Set of oids in the index that failed to load due to POSKeyError.
    # This is what happens if undo is applied to the transaction creating
    # the object (the oid is still in the index, but its current data
    # record has a backpointer of 0, and POSKeyError is raised then
    # because of that backpointer).
    undone = {}

    # Set of oids that were present in the index but failed to load.
    # This does not include oids in undone.
    noload = {}

    for oid in fs._index.keys():
        try:
            data, serial = fs.load(oid, "")
        except (KeyboardInterrupt, SystemExit):
            raise
        except POSKeyError:
            undone[oid] = 1
        except:
            if VERBOSE:
                traceback.print_exc()
            noload[oid] = 1

    inactive = noload.copy()
    inactive.update(undone)

    for oid in fs._index.keys():
        if oid in inactive:
            continue
        data, serial = fs.load(oid, "")
        refs = get_refs(data)
        missing = [] # contains 3-tuples of oid, klass-metadata, reason
        for info in refs:
            try:
                ref, klass = info
            except (ValueError, TypeError):
                # failed to unpack
                ref = info
                klass = '<unknown>'

            if ref in search_oids:
                report(oid, data, serial, [(ref, klass, "searching for")])
                
            if ref not in fs._index:
                missing.append((ref, klass, "missing"))
            if ref in noload:
                missing.append((ref, klass, "failed to load"))
            if ref in undone:
                missing.append((ref, klass, "object creation was undone"))
        if missing:
            report(oid, data, serial, missing)

# 需要导入模块: from ZODB.FileStorage import FileStorage [as 别名]
# 或者: from ZODB.FileStorage.FileStorage import load [as 别名]
    def test_pack_with_1_day(self):
        from ZODB.DB import DB
        from ZODB.FileStorage import FileStorage
        from ZODB.POSException import POSKeyError
        import time
        import transaction
        from relstorage.zodbpack import main

        storage = FileStorage(self.db_fn, create=True)
        db = DB(storage)
        conn = db.open()
        conn.root()['x'] = 1
        transaction.commit()
        oid = b('\0' * 8)
        state, serial = storage.load(oid, b(''))
        time.sleep(0.1)
        conn.root()['x'] = 2
        transaction.commit()
        conn.close()
        self.assertEqual(state, storage.loadSerial(oid, serial))
        db.close()
        storage = None

        main(['', '--days=1', self.cfg_fn])

        # packing should not have removed the old state.
        storage = FileStorage(self.db_fn)
        self.assertEqual(state, storage.loadSerial(oid, serial))
        storage.close()

# 需要导入模块: from ZODB.FileStorage import FileStorage [as 别名]
# 或者: from ZODB.FileStorage.FileStorage import load [as 别名]
    def generate_durus_object_records():
        sio = cStringIO.StringIO()
        zodb_storage = ZODBFileStorage(zodb_file_name)
        n = 0
        for oid in zodb_storage._index.keys():
            n += 1
            if n % 10000 == 0:
                sys.stdout.write('.')
                sys.stdout.flush()
            p, serial = zodb_storage.load(oid, '')
            refs = referencesf(p)
            # unwrap extra tuple from class meta data
            sio.seek(0)
            sio.write(p)
            sio.truncate()
            sio.seek(0)
            def get_class(module_class):
                module, klass = module_class
                if module not in sys.modules:
                    __import__(module)
                return getattr(sys.modules[module], klass)
            class PersistentRef:
                def __init__(self, v):
                    oid, module_class = v
                    self.oid_klass = (oid, get_class(module_class))
            unpickler = cPickle.Unpickler(sio)
            unpickler.persistent_load = lambda v: PersistentRef(v)
            class_meta = unpickler.load()
            class_meta, extra = class_meta
            assert extra is None
            object_state = unpickler.load()
            if type(object_state) == dict and  '_container' in object_state:
                assert 'data' not in object_state
                object_state['data'] = object_state['_container']
                del object_state['_container']
            sio.seek(0)
            sio.truncate()
            cPickle.dump(get_class(class_meta), sio, 2)

            pickler = cPickle.Pickler(sio, 2)
            def persistent_id(v):
                if isinstance(v, PersistentRef):
                    return v.oid_klass
                return None
            pickler.persistent_id = persistent_id
            pickler.dump(object_state)
            record = pack_record(oid, sio.getvalue(), ''.join(refs))
            yield record
        print
        print n, 'objects written'

# 需要导入模块: from ZODB.FileStorage import FileStorage [as 别名]
# 或者: from ZODB.FileStorage.FileStorage import load [as 别名]
def main(path):
    fs = FileStorage(path, read_only=1)
    if PACK:
        fs.pack()

    db = ZODB.DB(fs)
    rt = db.open().root()
    paths = find_paths(rt, 3)

    def total_size(oid):
        cache = {}
        cache_size = 1000

        def _total_size(oid, seen):
            v = cache.get(oid)
            if v is not None:
                return v
            data, serialno = fs.load(oid, "")
            size = len(data)
            for suboid in referencesf(data):
                if seen.has_key(suboid):
                    continue
                seen[suboid] = 1
                size += _total_size(suboid, seen)
            cache[oid] = size
            if len(cache) == cache_size:
                cache.popitem()
            return size

        return _total_size(oid, {})

    keys = fs._index.keys()
    keys.sort()
    keys.reverse()

    if not VERBOSE:
        # If not running verbosely, don't print an entry for an object
        # unless it has an entry in paths.
        keys = filter(paths.has_key, keys)

    fmt = "%8s %5d %8d %s %s.%s"

    for oid in keys:
        data, serialno = fs.load(oid, "")
        mod, klass = get_pickle_metadata(data)
        refs = referencesf(data)
        path = paths.get(oid, "-")
        print fmt % (U64(oid), len(data), total_size(oid), path, mod, klass)

# 需要导入模块: from ZODB.FileStorage import FileStorage [as 别名]
# 或者: from ZODB.FileStorage.FileStorage import load [as 别名]
def run(path, v=0):
    fs = FileStorage(path, read_only=1)
    # break into the file implementation
    if hasattr(fs._index, 'iterkeys'):
        iter = fs._index.iterkeys()
    else:
        iter = fs._index.keys()
    totals = {}
    for oid in iter:
        data, serialno = fs.load(oid, '')
        mod, klass = get_pickle_metadata(data)
        key = "%s.%s" % (mod, klass)
        bytes, count = totals.get(key, (0, 0))
        bytes += len(data)
        count += 1
        totals[key] = bytes, count
        if v:
            print "%8s %5d %s" % (U64(oid), len(data), key)
    L = totals.items()
    L.sort(lambda a, b: cmp(a[1], b[1]))
    L.reverse()
    print "Totals per object class:"
    for key, (bytes, count) in L:
        print "%8d %8d %s" % (count, bytes, key)