Python cPickle.Pickler类代码示例

class PickleItemExporter(BaseItemExporter):
    def __init__(self, file, protocol=0, **kwargs):
        self._configure(kwargs)
        self.pickler = Pickler(file, protocol)

    def export_item(self, item):
        self.pickler.dump(dict(self._get_serialized_fields(item)))

def pickle_matrices(matrices, outdir='.'):
	"""Pickles dictionary of matrices output by create_matrices"""
	for name, matrix in matrices.iteritems():
		fpath = os.path.join(outdir, name + '.pickle')
		with open(fpath, 'wb') as fh:
			pickler = Pickler(fh, HIGHEST_PROTOCOL)
			pickler.dump(matrix)

    def replicate(self, target):
        d = {'id': self.id,
             'user': self.user, 
             'description': self.description,
             'entries': self._entries}
        f = StringIO()
        p = Pickler(f)
        p.dump(d)

        payloadStr = f.getvalue()
        LOG('Replication', INFO, 'replicate> transaction id: %s; '
                'size (uncompressed): %s' % (
                    oid2str(self.id), len(payloadStr))) #DBG
        payloadStr = compress(payloadStr)

        handler = FixedHTTPHandler()
        opener = urllib2.build_opener(handler)
        urllib2.install_opener(opener)

        LOG('Replication', INFO, 'replicate> transaction id: %s; size: %s' % (
            oid2str(self.id), len(payloadStr))) #DBG
        url = '%s/load' % target.url
        schema, domain, path, x1, x2, x3 = urlparse.urlparse(url)
        newurl = '%s://%s:%[email protected]%s%s' % (
            schema, target.username, target.password, domain, path)
        try:
            urllib2.urlopen(newurl, urllib.urlencode({'data': payloadStr}))
        except urllib2.HTTPError, e:
            if e.code != 204: # 204 == 'No content' which is what we expect
                raise

 def SaveData(self, filename):
     gc.collect()
     self.Freeze()
     try:
         sdict = {"qList": copy.copy(self.qList),
                  "qRef": copy.copy(self.qRef),
                  "tagRef": self.tagRef,
                  "binStorage": self.binStorage,
                  "tempStorage": self.tempStorage,
                  "schedWatcher": self.schedWatcher,
                  "connManager": self.connManager}
         tmpFilename = filename + ".tmp"
         with open(tmpFilename, "w") as backup_printer:
             pickler = Pickler(backup_printer, 2)
             pickler.dump(sdict)
     except:
         logging.exception("Backup error")
     finally:
         self.UnFreeze()
     os.rename(tmpFilename, filename)
     if self.context.useMemProfiler:
         try:
             last_heap = self.LastHeap
             self.LastHeap = self.HpyInstance.heap()
             heapsDiff = self.LastHeap.diff(last_heap) if last_heap else self.LastHeap
             logging.info("memory changes: %s", heapsDiff)
             logging.debug("GC collecting result %s", gc.collect())
         except Exception, e:
             logging.exception("%s", e)

    def serialize(self, obj, deferred=False):
        """Serialize an object

        :param obj: The object to serialize.
        :param deferred: When this is true Deferred objects are
            serialized and their values are loaded on deserialization.
            When this is false Deferred objects are not serializable.
        """
        if deferred:
            args = {}

            def persistent_id(obj):
                if isinstance(obj, Deferred):
                    args[obj.id] = obj
                    return obj.id
                return None
        else:
            args = None

            def persistent_id(obj):
                if isinstance(obj, Deferred):
                    raise PicklingError('%s cannot be serialized' % obj)
                return None
        data = StringIO()
        pickle = Pickler(data, HIGHEST_PROTOCOL)
        pickle.persistent_id = persistent_id
        pickle.dump(obj)
        msg = data.getvalue()
        return (msg, args) if deferred else msg

def generate_episodedata_pickle(episodes):
    data = {}
    sys.stderr.write("Processed videos: ")
    for ep in episodes:
        epdata = {}
        epdata.update(ep)
        episode_url = "http://roosterteeth.com/archive/episode.php?id=%i" % ep["rtid"]
        blipid = get_blipid(episode_url)
        if blipid == None:
            # Youtube
            youtubeid = get_youtubeid(episode_url)
            if youtubeid != None:
                # Setup YouTube Service and get data
                youtube_service = gdata.youtube.service.YouTubeService()
                video_entry = youtube_service.GetYouTubeVideoEntry(video_id=youtubeid)
                epdata.update(parse_youtube(youtubeid, video_entry))
        else:
            page = urllib2.urlopen("http://blip.tv/rss/flash/%s" % blipid)
            epdata.update(parse_bliptv(page.read()))
            page.close()
        data[ep["rtid"]] = epdata
        sys.stderr.write("%s " % ep["rtid"])
    sys.stderr.write("\n")
    picklefile = StringIO()
    pickler = Pickler(picklefile, -1)
    pickler.dump(data)
    picklefile.seek(0)
    return picklefile

def save_game(username, save_obj):
    f = open(username + '.crsf','w')
    p = Pickler(f)
    p.dump(save_obj)
    f.close()
    
    pack_files(username)

 def __setitem__(self, key, value):
     f = StringIO()
     pickler = Pickler(f, self.PICKLING_PROTOCOL)
     pickler.dump(value)
     self._dict[str(key)] = f.getvalue()
     if hasattr(self._dict, 'sync'):
         self._dict.sync()

def cloneByPickle(obj, ignore_list=()):
    """Makes a copy of a ZODB object, loading ghosts as needed.

    Ignores specified objects along the way, replacing them with None
    in the copy.
    """
    ignore_dict = {}
    for o in ignore_list:
        ignore_dict[id(o)] = o

    def persistent_id(ob, ignore_dict=ignore_dict):
        if ignore_dict.has_key(id(ob)):
            return 'ignored'
        if getattr(ob, '_p_changed', 0) is None:
            ob._p_changed = 0
        return None

    def persistent_load(ref):
        assert ref == 'ignored'
        # Return a placeholder object that will be replaced by
        # removeNonVersionedData().
        placeholder = SimpleItem()
        placeholder.id = "ignored_subobject"
        return placeholder

    stream = StringIO()
    p = Pickler(stream, 1)
    p.persistent_id = persistent_id
    p.dump(obj)
    stream.seek(0)
    u = Unpickler(stream)
    u.persistent_load = persistent_load
    return u.load()

 def block_artist(self, artist_name):
     """store artist name and current daytime so songs by that
     artist can be blocked
     """
     self._blocked_artists.append(artist_name)
     self._blocked_artists_times.append(self.now)
     self.log("Blocked artist: %s (%s)" % (
         artist_name,
         len(self._blocked_artists)))
     if self.store_blocked_artists:
         dump = os.path.join(
             self.player_get_userdir(), "autoqueue_block_cache")
         try:
             os.remove(dump)
         except OSError:
             pass
     if len(self._blocked_artists) == 0:
         return
     if self.store_blocked_artists:
         pickle_file = open(dump, 'w')
         pickler = Pickler(pickle_file, -1)
         to_dump = (self._blocked_artists,
                    self._blocked_artists_times)
         pickler.dump(to_dump)
         pickle_file.close()

 def __setitem__(self, key, value):
     if self.writeback:
         self.cache[key] = value
     f = StringIO()
     p = Pickler(f, self._protocol)
     p.dump(value)
     self.dict[key] = f.getvalue()

    def __setitem__(self, key, value):
        with self._cache_write_lock:
            self._cache[key] = value

        f = StringIO()
        p = Pickler(f, self._protocol)
        p.dump(value)

        self._storage.redis.hset(self._hash_key, key, f.getvalue())

def deepcopy(obj):
    """Makes a deep copy of the object using the pickle mechanism.
    """
    stream = StringIO()
    p = Pickler(stream, 1)
    p.dump(aq_base(obj))
    stream.seek(0)
    u = Unpickler(stream)
    return u.load()

    def validate(self):
        if self.opt.verbose:
            print self.testname
        if not os.path.exists(os.path.join("..","jing")):
            print "Error: jing not found as sibling of processor archive."
            print "  Looked in: %s" % os.path.join("..","jing")
            sys.exit()
        m = re.match("(?sm).*version=\"([.0-9a-z]+)\".*",self.data["csl"])
        if m:
            rnc_path = os.path.join("csl","%s" % m.group(1), "csl.rnc")
        else:
            print "Error: Unable to find CSL version in %s" % self.hp
            sys.exit()
        tfd,tfilename = tempfile.mkstemp(dir=".")
        os.write(tfd,self.data["csl"])
        os.close(tfd)
        
        jfh = os.popen("java -jar %s -c %s %s" % (os.path.join("..","jing","bin","jing.jar"),rnc_path,tfilename))
        success = True
        plural = ""
        while 1:
            line = jfh.readline()
            if not line: break
            line = line.strip()
            e = re.match("^fatal:",line)
            if e:
                print line
                sys.exit()
            m = re.match(".*:([0-9]+):([0-9]+):  *error:(.*)",line)
            if m:
              if success:
                  print "\n##"
                  print "#### Error%s in CSL for test: %s" % (plural,self.hp)
                  print "##\n"
                  success = False
              print "  %s @ line %s" %(m.group(3).upper(),m.group(1))
              plural = "s"
        jfh.close()
        os.unlink(tfilename)
        if not success:
            print ""
            io = StringIO()
            io.write(self.data["csl"])
            io.seek(0)
            linepos = 1
            while 1:
                cslline = io.readline()
                if not cslline: break
                cslline = cslline.rstrip()
                print "%3d  %s" % (linepos,cslline)
                linepos += 1
            pfh = open( self.pickle,"w+b")
            pickler = Pickler( pfh )

            pickler.dump( (opt, self.pos) )
            sys.exit()

    def validate(self):
        if self.opt.verbose:
            print(self.testname)
        if not os.path.exists(self.cp.get("jing", "path")):
            print("Error: jing not found.")
            print("  Looked in: %s" % self.cp.get("jing", "path"))
            sys.exit()
        m = re.match("(?sm).*version=\"([.0-9a-z]+)\".*",self.data["csl"])
        if m:
            rnc_path = os.path.join(self.cp.get("csl", "v%s" % m.group(1)))
        else:
            print("Error: Unable to find CSL version in %s" % self.hp)
            sys.exit()
        tfd,tfilename = tempfile.mkstemp(dir=".")
        os.write(tfd,self.data["csl"].encode('utf8'))
        os.close(tfd)
        
        jfh = os.popen("%s %s -c %s %s" % (self.cp.get("jing", "command"), self.cp.get("jing", "path"),rnc_path,tfilename))
        success = True
        plural = ""
        while 1:
            line = jfh.readline()
            if not line: break
            line = line.strip()
            e = re.match("^fatal:",line)
            if e:
                print(line)
                sys.exit()
            m = re.match(".*:([0-9]+):([0-9]+):  *error:(.*)",line)
            if m:
              if success:
                  print("\n##")
                  print("#### Error%s in CSL for test: %s" % (plural,self.hp))
                  print("##\n")
                  success = False
              print("  %s @ line %s" %(m.group(3).upper(),m.group(1)))
              plural = "s"
        jfh.close()
        os.unlink(tfilename)
        if not success:
            print("")
            io = StringIO()
            io.write(self.data["csl"])
            io.seek(0)
            linepos = 1
            while 1:
                cslline = io.readline()
                if not cslline: break
                cslline = cslline.rstrip()
                print("%3d  %s" % (linepos,cslline))
                linepos += 1
            pfh = open( self.pickle,"wb+")
            pickler = Pickler( pfh )

            pickler.dump( (opt, self.pos) )
            sys.exit()