Python FileStorage.delete_file方法代码示例

# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import delete_file [as 别名]
def _spellchecker_for(word_set,
                      name,
                      spellcheck_cache_path=None,
                      sources=None):
    """Get a whoosh spellchecker for :word_set:.

    The word graph for this spellchecker will be stored on-disk with
    the unique-name :name: in :spellcheck_cache_path:, if it exists.
    This allows for much faster loading of word graphs after they have been
    pre-populated.

    :sources: is a list of filenames which will be checked to see if they
    are newer than the stored word graph. If they are newer, then the word
    graph gets repopulated.
    """
    assert "/" not in name and "\\" not in name

    if _spellchecker_cache.get(name, None) is not None:
        return _spellchecker_cache[name].corrector

    # Check the modification time of all the paths in :sources: to see
    # if they've been modified since the cache file was created. If so,
    # delete the cache file. This will cause it to be regenerated.
    #
    # Note that this relies on an implementation detail in whoosh, namely
    # that the cache file is always stored at spellcheck_cache_path/name.
    if spellcheck_cache_path:

        # Ensure that the directory has been created
        try:
            os.makedirs(spellcheck_cache_path)
        except OSError as error:
            if error.errno != errno.EEXIST:  # suppress(PYC90)
                raise error

        graph_path = os.path.realpath(spellcheck_cache_path)
        file_storage = FileStorage(graph_path)

        preexisting_cache = os.path.abspath(os.path.join(spellcheck_cache_path,
                                                         name))
        if os.path.exists(preexisting_cache):
            cache_mtime = os.path.getmtime(preexisting_cache)
            for source in sources:
                source_path = os.path.realpath(source)
                if not os.path.exists(source_path):
                    continue

                if os.path.getmtime(source_path) > cache_mtime:
                    file_storage.delete_file(name)
                    break

        try:
            word_graph = copy_to_ram(file_storage).open_file(name)
        except (IOError, NameError):
            word_graph = _create_word_graph_file(name, file_storage, word_set)

    else:
        ram_storage = RamStorage()
        word_graph = _create_word_graph_file(name, ram_storage, word_set)

    reader = fst.GraphReader(word_graph)
    corrector = spelling.GraphCorrector(reader)
    _spellchecker_cache[name] = SpellcheckerCacheEntry(corrector, reader)
    return corrector

# 需要导入模块: from whoosh.filedb.filestore import FileStorage [as 别名]
# 或者: from whoosh.filedb.filestore.FileStorage import delete_file [as 别名]
class TestReadWrite(unittest.TestCase):
    def __init__(self, *args, **kwargs):
        super(TestReadWrite, self).__init__(*args, **kwargs)
        self.fs = FileStorage(".")
    
    def make_postings(self):
        postings = [(1, 23), (3, 45), (12, 2), (34, 21), (43, 7), (67, 103), (68, 1), (102, 31),
                    (145, 4), (212, 9), (283, 30), (291, 6), (412, 39), (900, 50), (905, 28), (1024, 8),
                    (1800, 13), (2048, 3), (15000, 40)]
        return postings
    
    def make_file(self, name):
        return self.fs.create_file(name+"_test.pst")
    
    def open_file(self, name):
        return self.fs.open_file(name+"_test.pst")
    
    def delete_file(self, name):
        try:
            self.fs.delete_file(name+"_test.pst")
        except OSError:
            pass
    
    def test_readwrite(self):
        format = Frequency(None)
        postings = self.make_postings()
        
        postfile = self.make_file("readwrite")
        try:
            fpw = FilePostingWriter(postfile, blocklimit=8)
            fpw.start(format)
            for id, freq in postings:
                fpw.write(id, format.encode(freq))
            fpw.close()
            
            postfile = self.open_file("readwrite")
            fpr = FilePostingReader(postfile, 0, format)
            #self.assertEqual(postings, list(fpr.items_as("frequency")))
            fpr.close()
        finally:
            self.delete_file("readwrite")
        
    def test_skip(self):
        format = Frequency(None)
        postings = self.make_postings()
        
        postfile = self.make_file("skip")
        try:
            fpw = FilePostingWriter(postfile, blocklimit=8)
            fpw.start(format)
            for id, freq in postings:
                fpw.write(id, format.encode(freq))
            fpw.close()
            
            postfile = self.open_file("skip")
            fpr = FilePostingReader(postfile, 0, format)
            #fpr.skip_to(220)
            #self.assertEqual(postings[10:], list(fpr.items_as("frequency")))
            fpr.close()
        finally:
            self.delete_file("skip")
    
    def roundtrip(self, postings, format, astype):
        postfile = self.make_file(astype)
        readback = None
        try:
            fpw = FilePostingWriter(postfile, blocklimit=8)
            fpw.start(format)
            for id, value in postings:
                fpw.write(id, format.encode(value))
            fpw.close()
            
            postfile = self.open_file(astype)
            fpr = FilePostingReader(postfile, 0, format)
            readback = list(fpr.all_as(astype))
            fpr.close()
        finally:
            self.delete_file(astype)
        return readback
    
    def test_existence_postings(self):
        postings = []
        docnum = 0
        for _ in xrange(0, 20):
            docnum += randint(1, 10)
            postings.append((docnum, 1))
        
        self.assertEqual(postings, self.roundtrip(postings, Existence(None), "frequency"))
    
    def test_docboost_postings(self):
        postings = []
        docnum = 0
        for _ in xrange(0, 20):
            docnum += randint(1, 10)
            freq = randint(1, 1000)
            boost = byte_to_float(float_to_byte(random() * 2))
            postings.append((docnum, (freq, boost)))
        
        self.assertEqual(postings, self.roundtrip(postings, DocBoosts(None), "docboosts"))
        
#.........这里部分代码省略.........