Python ServerProxy.getIndexedFiles方法代码示例

# 需要导入模块: from xmlrpclib import ServerProxy [as 别名]
# 或者: from xmlrpclib.ServerProxy import getIndexedFiles [as 别名]
class Indexer(AbstractIndexer):
    """An Indexer instance periodically looks in the configured
    directories for files to index If it detects changes in known
    files, it sends a request to the Querier (via xmlrpc) to index the
    file, giving the Querier information on the file. The querier may
    decide to do nothing if it detects that the database is up-to-date.
    """
    
    def __init__(self, indexerConfig, observers=None):
        username = indexerConfig.user
        password = indexerConfig.password
        host = indexerConfig.host
        port = indexerConfig.port
        print "Indexer connecting to Node %s:%s" % (host, port)        
        self.serverProxy = ServerProxy('http://%s:%s' % (host, port),
                                       allow_none=True,
                                       encoding='utf-8')
        self.cnxId, errmsg = self.serverProxy.authenticate(username, password)
        if not self.cnxId:
            if self.verbose:
                print "Got failure from Node:", errmsg
            raise MaayAuthenticationError("Failed to connect as '%s'" % username)
        # baseclass's __init__ must be called *after* local initialisation
        # otherwise it could call _purgeEverything with an inconsistent state
        AbstractIndexer.__init__(self, indexerConfig, observers)
        
    def purgeFiles(self,fileset):
        for filename in fileset:
            if self.verbose:
                print "Requesting unindexation of %s" % \
                      safe_encode(filename)
            self.serverProxy.removeFileInfo(self.cnxId, filename)
        if self.verbose:
            print "Requesting cleanup of unreferenced documents"
        self.serverProxy.removeUnreferencedDocuments(self.cnxId)

    def _purgeEverything(self):
        indexedFiles = set(self.serverProxy.getIndexedFiles(self.cnxId))
        self.purgeFiles(indexedFiles)

    def _getIndexedFiles(self):
        return set(self.serverProxy.getIndexedFiles(self.cnxId))
       
    def getLastIndexationTimeAndState(self, filename):
        answer = self.serverProxy.lastIndexationTimeAndState(self.cnxId, filename)
        if answer is None:
            raise MaayAuthenticationError("Bad cnxId sent to the Node")
        lastTime, lastState = answer
        return lastTime, lastState

    def indexDocument(self, futureDoc):
        futureDoc.file_state=FileInfo.CREATED_FILE_STATE
        if self.verbose:
            print "Requesting indexation of %s" % \
                  safe_encode(futureDoc.filename),
        try:
            futureDoc.title = removeControlChar(futureDoc.title) 
            futureDoc.text = removeControlChar(futureDoc.text)
            if self.verbose:
                print '('+safe_encode(futureDoc.title)+')'
            self.serverProxy.indexDocument(self.cnxId, futureDoc)
        except (Fault, ProtocolError), exc:
            if self.verbose:
                print "An error occured on the Node while indexing %s" % \
                      safe_encode(futureDoc.filename)
                print exc
                print "See Node log for details"
            else:
                print "Error indexing %s: %s" % \
                      (safe_encode(futureDoc.filename), exc)
        else:

# 需要导入模块: from xmlrpclib import ServerProxy [as 别名]
# 或者: from xmlrpclib.ServerProxy import getIndexedFiles [as 别名]
class Indexer:
    """An Indexer instance periodically looks in the configured
    directories for files to index If it detects changes in known
    files, it sends a request to the Querier (via xmlrpc) to index the
    file, giving the Querier information on the file. The querier may
    decide to do nothing if it detects that the database is up-to-date.
    """
    
    def __init__(self, indexerConfig):
        self.indexerConfig = indexerConfig
        username = self.indexerConfig.user
        password = self.indexerConfig.password
        host = self.indexerConfig.host
        port = self.indexerConfig.port
        self.filesystemEncoding = sys.getfilesystemencoding()
        print "Indexer connecting to server %s:%s" % (host, port)
        self.serverProxy = ServerProxy('http://%s:%s' % (host, port),
                                       allow_none=True,
                                       encoding='utf-8')
        self.cnxId, errmsg = self.serverProxy.authenticate(username, password)
        self.verbose = indexerConfig.verbose
        if not self.cnxId:
            if self.verbose:
                print "Got failure from server:", errmsg
            raise MaayAuthenticationError("Failed to connect as '%s'" % username)
        # we might be asked to purge everything and just exit
        if indexerConfig['purge']:
            self._purgeEverything()
            sys.exit(0)
            
        
    def getFileIterator(self, isPrivate=True):
        if isPrivate:
            indexed = self.indexerConfig.private_index_dir
            skipped = self.indexerConfig.private_skip_dir
            print "private indexation of", indexed, "omitting", skipped
        else:
            indexed = self.indexerConfig.public_index_dir
            skipped = self.indexerConfig.public_skip_dir
            print "public indexation of", indexed, "omitting", skipped
        return FileIterator(indexed, skipped)

    def isIndexable(self, filename):
        return converter.isKnownType(filename)

    def purgeFiles(self,fileset):
        for filename in fileset:
            if self.verbose:
                print "Requesting unindexation of %s" % filename
            self.serverProxy.removeFileInfo(self.cnxId,
                                            unicode(filename,
                                                    self.filesystemEncoding))
        if self.verbose:
            print "Requesting cleanup of unreferenced documents"
        self.serverProxy.removeUnreferencedDocuments(self.cnxId)

    def _purgeEverything(self):
        indexedFiles = Set(self.serverProxy.getIndexedFiles(self.cnxId))
        self.purgeFiles(indexedFiles)

    def start(self):
        # we index private dirs first because public overrides private
        existingFiles = self.runIndexer(isPrivate=True)
        existingFiles |= self.runIndexer(isPrivate=False)
        indexedFiles = Set(self.serverProxy.getIndexedFiles(self.cnxId))
        oldFiles = indexedFiles - existingFiles
        self.purgeFiles(oldFiles)

    def runIndexer(self, isPrivate=True):
        existingFiles = Set()
        
        if isPrivate:
            state = Document.PRIVATE_STATE
        else:
            state = Document.PUBLISHED_STATE
            
        for filename in self.getFileIterator(isPrivate):
            existingFiles.add(filename)
            if not self.isIndexable(filename):
                continue
            lastModificationTime = os.path.getmtime(filename)
            lastIdxTime, lastIdxState = self.getLastIndexationTimeAndState(filename)
            if lastIdxState == state and lastIdxTime >= lastModificationTime:
                if self.verbose:
                    print "%s didn't change since last indexation" % filename
                continue
            else:
                fileSize = os.path.getsize(filename)
                try:
                    title, text, links, offset = converter.extractWordsFromFile(filename)
                except converter.IndexationFailure, exc:
                    if self.verbose:
                        print exc
                    continue
                docId = makeDocumentId(filename)
                mime_type = mimetypes.guess_type(filename)[0]

                self.indexDocument(FutureDocument(filename=unicode(filename,
                                                                   self.filesystemEncoding),
                                                  title=title, text=text,
#.........这里部分代码省略.........