Python index.create_in函数代码示例

 def open_index(self, schema):
     """ Opens an index. Returns the writer. """
     if not os.path.exists(self.index_path):
         os.mkdir(self.index_path)
         index.create_in(self.index_path, schema)
     self._index = index.open_dir(self.index_path)
     return self._index.writer()

	def run(self):
		# open index
		self.buffer = deque(maxlen=BUFFERLINES)
		if not exists(self.indexdir):
			makedirs(self.indexdir)
			self.ix = create_in(self.indexdir, SCHEMA)
		else:
			if exists_in(self.indexdir): self.ix = open_dir(self.indexdir)
			else: self.ix = create_in(self.indexdir, SCHEMA)
		self.qp = QueryParser("content", self.ix.schema)
		self.searcher = self.ix.searcher()
		index_p = self.index_p
		while True:
			try:
				# check index_p
				try:
					type, data = index_p.recv()
				except EOFError: break
				try:
					if type == QUERY: self._processSearch(data)
					elif type == LOG: self._processLog(data)
					elif type == RENAME: self._processRename(data)
					else:
						prnt("Unexpected data in logindexsearch.")
				except:
					print_exc()
					prnt("EXCEPTION in logindexsearch process.")
			except KeyboardInterrupt:
				break
		self._dumpBuffer(self.buffer)
		self.searcher.close()
		self.ix.close()	

 def __load__(region=None):
     """加载/建立索引
     :param region: 索引范围，None表示加载所有索引；news\blog表示加载对应索引
     :return: 是否加载成功
     """
     # 加载索引
     if region:
         if region in Indexer.__index__:
             return True
         else:
             if region not in index_dir:
                 return False
             if not os.path.exists(index_dir[region]):
                 os.makedirs(index_dir[region])
                 Indexer.__index__[region] = index.create_in(index_dir[region], schema, indexname=region)
             else:
                 Indexer.__index__[region] = index.open_dir(index_dir[region], indexname=region)
             return True
     else:  # 加载全部索引
         for reg in index_dir.keys():
             if reg in Indexer.__index__:
                 return True
             else:
                 if not os.path.exists(index_dir[reg]):
                     os.mkdir(index_dir[reg])
                     Indexer.__index__[reg] = index.create_in(index_dir[reg], schema, indexname=reg)
                 else:
                     Indexer.__index__[reg] = index.open_dir(index_dir[reg], indexname=reg)
                 return True

    def create_whoosh(self):
        print "creating_whoosh: "

        #initlock = lockfile.ThreadSafeFile(WHOOSH_FOLDER, '_init')
        thistime = datetime.datetime.now()
        dateformat = '%d-%m-%Y %H:%M:%S'
        create_index_flag = False
        #try: 
        #    initlock.acquire(timeout=2)
        #except lockfile.LockTimeout:
        #    print "Lock timeout when trying to create whoosh index schema. Continuing without index creation"
        #    return
        #except lockfile.AlreadyLocked:
        #    print "Already locked. Continuing without index creation"                
        #    return
        try:
                last_creation = datetime.datetime.strptime(initlock.read(), dateformat) #deserialize 
                print "Last index creation: %s"%datetime.datetime.strftime(last_creation, '%d-%m-%Y %H:%M:%S')
                if (thistime - last_creation).total_seconds() > 4*60*60: #4 hours  
                        create_index_flag = True
                        print "Index older than 4 hours - will recreate"
                else: print "Index is fresh - will not recreate"
        except: create_index_flag = True #do the creation anyway, maybe initial condition
        if create_index_flag:
            create_in(WHOOSH_FOLDER, schema)
            print "Creating search index"
            
            writer = ix.writer()
            for t in self.collector:
                #print "index: Adding term %s"%t[0]
                writer.add_document(term=u"%s"%t[0], url=u"%s"%t[1], description=u"%s"%t[2])
            writer.commit()
            #we can free now the collector
            self.collector = None

def build_index():
    print("build index")
    client = MongoClient('localhost', 27017)
    collection = client['bdhackathon']['Japan_Travel']
    schema = Schema(
        article_title=TEXT(stored=True, analyzer=analyzer), 
        article_id=TEXT(stored=True),
        author=TEXT(stored=True),
        #content=TEXT(stored=True, analyzer=analyzer)
        )

    #Initial Whoosh index
    if not os.path.exists("index"):
        os.mkdir("index")
        create_in("index", schema)

    ix = open_dir("index")
    writer = ix.writer()
    articles = collection.find()
    for article in articles:
        writer.update_document(
            article_title = article["article_title"],
            article_id = article["article_id"],
            author = article["author"]["account"],
            #content= article["content"]
            )
    writer.commit()

    def build_search(cls):
        analyzer = cls.analyzer

        schema = Schema(
            nid=ID(unique=True, stored=True),
            slug=ID(unique=True, stored=True),
            title=TEXT(stored=True, analyzer=analyzer),
            tag=KEYWORD(stored=True, lowercase=True, commas=True,
                        scorable=True),
            description=TEXT(stored=True, analyzer=analyzer),
            content=TEXT(stored=True, analyzer=analyzer)
        )

        folder = cls.tmp_dir
        if not os.path.exists(folder):
            os.mkdir(folder)
        create_in(folder, schema)

        ix = open_dir(folder)
        writer = ix.writer()

        for article in Article.find({'status': Article.ACCEPTED}):
            writer.update_document(
                nid=str(article._id),
                slug=article.slug,
                title=article.title,
                tag=','.join(article.tag),
                description=article.description,
                content=article.content
            )

        writer.commit()

        cls.searcher = ix.searcher()

    def build_indexes(self):
        if os.path.exists(self.index_location):
            log.debug('removing previous index')
            rmtree(self.index_location)

        if not os.path.exists(self.index_location):
            os.mkdir(self.index_location)

        chgset_idx = create_in(self.index_location, CHGSETS_SCHEMA,
                               indexname=CHGSET_IDX_NAME)
        chgset_idx_writer = chgset_idx.writer()

        file_idx = create_in(self.index_location, SCHEMA, indexname=IDX_NAME)
        file_idx_writer = file_idx.writer()
        log.debug('BUILDING INDEX FOR EXTENSIONS %s '
                  'AND REPOS %s' % (INDEX_EXTENSIONS, self.repo_paths.keys()))

        for repo_name, repo in self.repo_paths.items():
            # skip indexing if there aren't any revisions
            if len(repo) < 1:
                continue

            self.index_files(file_idx_writer, repo_name, repo)
            self.index_changesets(chgset_idx_writer, repo_name, repo)

        log.debug('>> COMMITING CHANGES <<')
        file_idx_writer.commit(merge=True)
        chgset_idx_writer.commit(merge=True)
        log.debug('>>> FINISHED BUILDING INDEX <<<')

 def setup(self):
     """
     Defers loading until needed.
     """
     new_index = False
     
     # Make sure the index is there.
     if not os.path.exists(settings.HAYSTACK_WHOOSH_PATH):
         os.makedirs(settings.HAYSTACK_WHOOSH_PATH)
         new_index = True
     
     self.storage = store.FileStorage(settings.HAYSTACK_WHOOSH_PATH)
     self.content_field_name, fields = self.site.build_unified_schema()
     self.schema = self.build_schema(fields)
     self.parser = QueryParser(self.content_field_name, schema=self.schema)
     
     if new_index is True:
         self.index = index.create_in(settings.HAYSTACK_WHOOSH_PATH, self.schema)
     else:
         try:
             self.index = index.Index(self.storage, schema=self.schema)
         except index.EmptyIndexError:
             self.index = index.create_in(settings.HAYSTACK_WHOOSH_PATH, self.schema)
     
     self.setup_complete = True

    def setup(self):
        """
        Defers loading until needed.
        """
        new_index = False

        # Make sure the index is there.
        if not os.path.exists(settings.HAYSTACK_WHOOSH_PATH):
            os.makedirs(settings.HAYSTACK_WHOOSH_PATH)
            new_index = True

        if not os.access(settings.HAYSTACK_WHOOSH_PATH, os.W_OK):
            raise IOError(
                "The path to your Whoosh index '%s' is not writable for the current user/group."
                % settings.HAYSTACK_WHOOSH_PATH
            )

        self.storage = FileStorage(settings.HAYSTACK_WHOOSH_PATH)
        self.content_field_name, self.schema = self.build_schema(self.site.all_searchfields())
        self.parser = QueryParser(self.content_field_name, schema=self.schema)

        if new_index is True:
            self.index = index.create_in(settings.HAYSTACK_WHOOSH_PATH, self.schema)
        else:
            try:
                self.index = self.storage.open_index(schema=self.schema)
            except index.EmptyIndexError:
                self.index = index.create_in(settings.HAYSTACK_WHOOSH_PATH, self.schema)

        self.setup_complete = True

    def clear_index(self):
        """
        Clear index: whoosh indexe create, create a new index in the directory
        even if an index exists.
        """

        if os.path.exists("indexes"):
            index.create_in("indexes", self.schema)

def main(script, command='', index='', field='', *query):
    """Store, clear or search data in whoosh indices.

    Can also be used to create vectors needed for task 3.
    'command' is either build|store|clean|search|vector
    'index' is either atc|icd|therapy|case

    Usage: python3 index.py <command> [index] [field] [query]
    """
    # Store all objects in index
    if command == 'build':
        populate_all()
        empty = get_empty_indices()
        for cls in empty:
            store_objects_in_index(cls)
        return

    classes = [ATC, ICD, PatientCase, Therapy]
    if index:
        classes = [i for i in classes if i._NAME == index]
        if not classes:
            print("Unknown index %s, valids: atc|icd|case|therapy" % index)
            sys.exit(2)

    # Store objects in index, will create duplicates if run several times
    if command == 'store':
        populate_all()
        for cls in classes:
            store_objects_in_index(cls)

    # Empty index
    elif command in ('clean', 'clear'):
        for cls in classes:
            create_or_open_index(cls)
            create_in(INDEX_DIR, SCHEMA_MAP[cls._NAME], cls._NAME)
            print("Emptied %s index" % cls.__name__)

    # Create vectors
    elif command.startswith('vector'):
        populate_all()
        create_vectors()

    # Search in whoosh index
    elif command == 'search':
        mapping = {'icd': ('short', 'label'), 'atc': ('code', 'title'),
                   'therapy': ('code', 'title'), 'case': ('code',)}
        query = ''.join(query)  # Flatten query
        cls, = classes  # Can only search on one index at a time
        print_result(extract(mapping[cls._NAME], search(cls, field, query)))

    # Unknown command
    else:
        print("Unknown command '%s'" % command)
        print("Usage: python3 index.py <command> [index] [field] [query]")
        print("Command is either build|store|clean|search|vector")
        sys.exit(2)

    sys.exit(None)

 def setup(self):
     import os
     if not os.path.exists(self.location):
         os.mkdir(self.location)
         self.ix = index.create_in(self.location, self.schema)
     elif index.exists_in(self.location):
         self.ix = index.open_dir(self.location, schema=self.schema)
     else:
         self.ix = index.create_in(self.location, self.schema)

 def get_index(self):
     ip = self.indexpath
     if not self.indexpath.startswith('/'):
         ip = path.join(self.env.path, ip)
     if not path.exists(ip):
         os.mkdir(ip)
     if not index.exists_in(ip):
         index.create_in(ip, self.SCHEMA)
     return index.open_dir(ip)

 def create(self, path):
     """
     Create the index directory if it hasn't already been created.
     """
     if not os.path.exists(path):
         os.mkdir(path)
         create_in(self.index_path, self.schema)
         return True
     return False

    def test_detects_that_index_needs_upgrade(self):
        wrong_schema = Schema(content=TEXT())
        index.create_in(self.index_dir, schema=wrong_schema)

        whoosh_backend = WhooshBackend(self.env)
        self.assertEqual(whoosh_backend.is_index_outdated(), True)

        whoosh_backend.recreate_index()
        self.assertEqual(whoosh_backend.is_index_outdated(), False)