Python marshal.load函数代码示例

def remove_duplicates_randomized():
    try:
        docs = m.load(open('../input/reuters/docs.p','rb'))
        docs_paths = m.load(open('../input/reuters/docs_paths.p','rb'))
    except:
        print 'No serialized file...'
        path = "../input/reuters/output"
        dirs = os.listdir(path)
        docs = []
        docs_paths = []
        for d in dirs:
            dirpath = os.path.join(path,d)
            files = os.listdir(dirpath)
            docs += files
            files = [os.path.join(dirpath,f) for f in files]
            docs_paths += files

        m.dump(docs,open("../input/reuters/docs.p","rb"))
        m.dump(docs_paths,open("../input/reuters/docs_paths.p","rb"))

    d = Counter(docs)
    no_keys = len(d.keys())
    counter = 0
    docs_arr = array(docs)
    for doc in d.keys():
        counter += 1
        if counter % 100 == 0:
            print "processed %d of %d"%(counter,no_keys)
        if d[doc] > 1:
            indices = list(where(docs_arr == doc)[0])
            # randomly remove the duplicates
            for i in range(len(indices)-1):
                idx = choice(indices)
                indices.remove(idx)
                os.remove(docs_paths[idx])

 def get_code(self,fullname):
     """Get the code object for the given module."""
     file,pathname,description = self._get_module_info(fullname)
     if file is not None:
         file.close()
     if description[2] == imp.PKG_DIRECTORY:
         for (suffix,_,typ) in imp.get_suffixes():
             if typ != imp.PY_COMPILED:
                 continue
             initfile = os.path.join(pathname,"__init__"+suffix)
             if os.path.exists(initfile):
                 f = open(initfile,"rb")
                 try:
                     f.seek(8)
                     return marshal.load(f)
                 finally:
                     f.close()
         return self.get_code(fullname+".__init__")
     else:
         pathbase = pathname[:-1*len(description[0])]
         for (suffix,_,typ) in imp.get_suffixes():
             if typ != imp.PY_COMPILED:
                 continue
             codefile = pathbase+suffix
             if os.path.exists(codefile):
                 f = open(codefile,"rb")
                 try:
                     f.seek(8)
                     return marshal.load(f)
                 finally:
                     f.close()
         source = self.get_source(fullname)
         if source is not None:
             return compile(source,pathname,"exec")
     return None

def load_model(f_name):
    _curpath = os.path.normpath(
        os.path.join(os.getcwd(), os.path.dirname(__file__)))
    # For Jython
    start_p = {}
    abs_path = os.path.join(_curpath, PROB_START_P)
    with open(abs_path, 'rb') as f:
        start_p = marshal.load(f)

    trans_p = {}
    abs_path = os.path.join(_curpath, PROB_TRANS_P)
    with open(abs_path, 'rb') as f:
        trans_p = marshal.load(f)

    emit_p = {}
    abs_path = os.path.join(_curpath, PROB_EMIT_P)
    with open(abs_path, 'rb') as f:
        emit_p = marshal.load(f)

    state = {}
    abs_path = os.path.join(_curpath, CHAR_STATE_TAB_P)
    with open(abs_path, 'rb') as f:
        state = marshal.load(f)
    f.closed

    return state, start_p, trans_p, emit_p, result

 def charge_et_genere_3d(self,fichier):
     #print marshal.version
     f = open(fichier,'rb')
     self.section_catalogue=marshal.load(f)
     obj=marshal.load(f)
     f.close()
     
     formecentre=self.__salome_genere_section_catalogue()
     
     newvolume=[]
     repereglobal= geompy.MakeMarker(0,0,0, 1,0,0,0,1,0)
     for a in obj:
         ori=geompy.MakeVertex(a[0][0],a[0][1],a[0][2])
         fin=geompy.MakeVertex(a[1][0],a[1][1],a[1][2])
         nouveaurepere= geompy.MakeMarker(a[0][0],a[0][1],a[0][2], a[2][0],a[2][1],a[2][2],a[3][0],a[3][1],a[3][2])
         basesurface=geompy.MakePosition(formecentre[a[4]],repereglobal,nouveaurepere)
         newvolume.append(geompy.MakePrism(basesurface,ori,fin))
         
     #for k,v in formecentre.iteritems():
     #    geompy.addToStudy(v,k)
     #for v in newvolume:
     #    geompy.addToStudy(v,"Volumique")
     outshape=geompy.MakeCompound(newvolume)
     geompy.addToStudy(outshape,"Volumique")
     #geompy.Export(outshape, "/home/fred/asteretude/kuwait/toto.stp", "STEP")
     salome.sg.updateObjBrowser(1)

def load_original_char_table(selected_files):
	"""load char_table from disk"""

	if selected_files is None or len(selected_files) == 0:
		selected_files = [os.path.join(os.getcwd(), dat_dir, f) for f in os.listdir(dat_dir) if os.path.isfile(os.path.join(dat_dir,f)) and f.endswith('.dat')]

	char_table = {}

	for datfile in selected_files:
		print "load %s" % datfile
		inf = open(datfile, 'rb')
		try:
			text_title = marshal.load(inf)
			new_char_table = marshal.load(inf)

			# merge new_char_table to char_table
			for key, value in new_char_table.items():
				if char_table.has_key(key):
					char_table[key]['context'] += value['context']
					char_table[key]['rating'] = max(char_table[key]['rating'], value['rating'])
					char_table[key]['tabu'] = char_table[key]['tabu'].union(value['tabu'])
				else:
					char_table[key] = value
		finally:
			inf.close()

	return char_table

    def test_floats(self):
        # Test a few floats
        small = 1e-25
        n = sys.maxint * 3.7e250
        while n > small:
            for expected in (-n, n):
                f = float(expected)
                s = marshal.dumps(f)
                got = marshal.loads(s)
                self.assertEqual(f, got)
                marshal.dump(f, file(test_support.TESTFN, "wb"))
                got = marshal.load(file(test_support.TESTFN, "rb"))
                self.assertEqual(f, got)
            n /= 123.4567

        f = 0.0
        s = marshal.dumps(f)
        got = marshal.loads(s)
        self.assertEqual(f, got)

        n = sys.maxint * 3.7e-250
        while n < small:
            for expected in (-n, n):
                f = float(expected)
                s = marshal.dumps(f)
                got = marshal.loads(s)
                self.assertEqual(f, got)
                marshal.dump(f, file(test_support.TESTFN, "wb"))
                got = marshal.load(file(test_support.TESTFN, "rb"))
                self.assertEqual(f, got)
            n *= 123.4567
        os.unlink(test_support.TESTFN)

    def charge_et_genere_3d(self,fichier,exportstl=None):
        #print marshal.version

        f = open(fichier,'rb')
        self.section_catalogue=marshal.load(f)
        obj=marshal.load(f)
        f.close()
        
        formecentre=self.__salome_genere_section_catalogue()
        
        newvolume=[]
        repereglobal= geompy.MakeMarker(0,0,0, 1,0,0,0,1,0)
        for a in obj:
            #impression du nom de la forme debug
            #print a[4]
            
            ori=geompy.MakeVertex(a[0][0],a[0][1],a[0][2])
            fin=geompy.MakeVertex(a[1][0],a[1][1],a[1][2])
            nouveaurepere= geompy.MakeMarker(a[0][0],a[0][1],a[0][2], a[2][0],a[2][1],a[2][2],a[3][0],a[3][1],a[3][2])
            basesurface=geompy.MakePosition(formecentre[a[4]],repereglobal,nouveaurepere)
            newvolume.append(geompy.MakePrism(basesurface,ori,fin))
            
        #for k,v in formecentre.iteritems():
        #    geompy.addToStudy(v,k)
        #for v in newvolume:
        #    geompy.addToStudy(v,"Volumique")
        outshape=geompy.MakeCompound(newvolume)

        geompy.addToStudy(outshape,"Volumique")

        if exportstl!=None:
           geompy.Export(outshape, exportstl, "STL_Bin")

        salome.sg.updateObjBrowser(1)

    def __init__(self, settings):
        super(PyborgBrain, self).__init__(settings)

        self.log.info("Reading dictionary...")
        try:
            zfile = zipfile.ZipFile('archive.zip', 'r')
        except (EOFError, IOError):
            self.log.debug("No archive.zip found to unarchive")
        else:
            # Unarchive all the files from the zip.
            for filename in zfile.namelist():
                data = zfile.read(filename)
                with open(filename, 'w+b') as data_file:
                    data_file.write(data)

        try:
            with open('version', 'rb') as version_file:
                content = version_file.read()
            if content != self.saves_version:
                self.log.error("Dictionary is version %s but version %s is required. Please convert the dictionary.",
                    content, self.saves_version)
                # TODO: use an exception here
                sys.exit(1)

            with open('words.dat', 'rb') as words_file:
                self.words = marshal.load(words_file)
            with open('lines.dat', 'rb') as lines_file:
                self.lines = marshal.load(lines_file)
        except (EOFError, IOError):
            self.log.info("Couldn't read saved dictionary, so using a new database.")
            self.words = {}
            self.lines = {}

        self.num_words = len(self.words)
        self.num_contexts = sum(len(line[0].split()) for line in self.lines.itervalues())

        self.log.debug("Checking dictionary for new aliases...")
        for word in self.words.keys():
            if word.startswith('~'):
                if word not in self.settings.aliases:
                    self.log.debug("Unlearning alias %r", word)
                    self.unlearn_word(word)
            else:
                for alias_word, patterns in self.settings.aliases.iteritems():
                    for alias_pattern in patterns:
                        pattern = r'^%s$' % alias_pattern
                        if re.search(pattern, word):
                            self.log.debug("Discovered alias %r for word %r, replacing", alias_word, word)
                            self.replace_word(word, alias_word)

        # Unlearn words in the unlearn.txt file.
        try:
            with open('unlearn.txt', 'r') as unlearn_file:
                for word in unlearn_file:
                    word = word.strip()
                    if word and word in self.words:
                        self.unlearn_word(word)
        except (EOFError, IOError):
            # No words to unlearn.
            pass

 def test_dict(self):
     new = marshal.loads(marshal.dumps(self.d))
     self.assertEqual(self.d, new)
     marshal.dump(self.d, file(test_support.TESTFN, "wb"))
     marshal.load(file(test_support.TESTFN, "rb"))
     self.assertEqual(self.d, new)
     os.unlink(test_support.TESTFN)

def scan_module(egg_dir, base, name, stubs):
    """Check whether module possibly uses unsafe-for-zipfile stuff"""

    filename = os.path.join(base,name)
    if filename[:-1] in stubs:
        return True     # Extension module
    pkg = base[len(egg_dir)+1:].replace(os.sep,'.')
    module = pkg+(pkg and '.' or '')+os.path.splitext(name)[0]
    f = open(filename,'rb'); f.read(8)   # skip magic & date
    try:
        code = marshal.load(f); f.close()
    except ValueError:
        f.seek(0); f.read(12)  # skip magic & date & file size; file size added in Python 3.3
        code = marshal.load(f); f.close()
    safe = True
    symbols = dict.fromkeys(iter_symbols(code))
    for bad in ['__file__', '__path__']:
        if bad in symbols:
            log.warn("%s: module references %s", module, bad)
            safe = False
    if 'inspect' in symbols:
        for bad in [
            'getsource', 'getabsfile', 'getsourcefile', 'getfile'
            'getsourcelines', 'findsource', 'getcomments', 'getframeinfo',
            'getinnerframes', 'getouterframes', 'stack', 'trace'
        ]:
            if bad in symbols:
                log.warn("%s: module MAY be using inspect.%s", module, bad)
                safe = False
    if '__name__' in symbols and '__main__' in symbols and '.' not in module:
        if sys.version[:3]=="2.4":  # -m works w/zipfiles in 2.5
            log.warn("%s: top-level module may be 'python -m' script", module)
            safe = False
    return safe

 def test_list(self):
     lst = self.d.items()
     new = marshal.loads(marshal.dumps(lst))
     self.assertEqual(lst, new)
     marshal.dump(lst, file(test_support.TESTFN, "wb"))
     marshal.load(file(test_support.TESTFN, "rb"))
     self.assertEqual(lst, new)
     os.unlink(test_support.TESTFN)

 def test_tuple(self):
     t = tuple(self.d.keys())
     new = marshal.loads(marshal.dumps(t))
     self.assertEqual(t, new)
     marshal.dump(t, file(test_support.TESTFN, "wb"))
     marshal.load(file(test_support.TESTFN, "rb"))
     self.assertEqual(t, new)
     os.unlink(test_support.TESTFN)

def getIDFScores(log = False):
  if log:
    with open('idf_dict_log', 'rb') as f:
      idf_dict = marshal.load(f)
  else:
    with open('idf_dict', 'rb') as f:
      idf_dict = marshal.load(f)
  return idf_dict

def read_results():
    searchresults_file = open(filename1, 'rb')
    search_results = marshal.load(searchresults_file)
    searchresults_file.close()
    searchresults_file = open(filename2, 'rb')
    topic = marshal.load(searchresults_file)
    searchresults_file.close()
    return search_results, topic

    def read(path):
        log.info('Reading entity cooccurrence model from file: %s' % path)

        with open(path, 'rb') as f:
            cooccurrence_counts = marshal.load(f)
            occurrence_counts = marshal.load(f)

        return EntityCooccurrence(cooccurrence_counts, occurrence_counts)