当前位置: 首页>>代码示例>>Python>>正文


Python DbClient.getPage方法代码示例

本文整理汇总了Python中dbclient.DbClient.getPage方法的典型用法代码示例。如果您正苦于以下问题:Python DbClient.getPage方法的具体用法?Python DbClient.getPage怎么用?Python DbClient.getPage使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在dbclient.DbClient的用法示例。


在下文中一共展示了DbClient.getPage方法的3个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。

示例1: main

# 需要导入模块: from dbclient import DbClient [as 别名]
# 或者: from dbclient.DbClient import getPage [as 别名]
def main():
     
     collectionName = "jobinfo_se_top_corps"     
     dbClient = DbClient('localhost', 27017, "jobaly")
     collection = dbClient.getCollection(collectionName)
    
     pageSize = 100 
     pageNo = 1
     has_more = True
     pageNum = 10000
     find_sort = None
     find_spec=None
     while has_more and pageNo <= pageNum :
        page = dbClient.getPage(collection, find_spec,find_sort, pageSize, pageNo)    
        processPage(collection, page,pageNo)        
        pageNo+=1 
        count =  page.count(with_limit_and_skip = True)
     #   print "count=",count
        if ( count < pageSize ) :
            has_more = False
开发者ID:folagit,项目名称:resumatcher,代码行数:22,代码来源:datawasher.py

示例2: main

# 需要导入模块: from dbclient import DbClient [as 别名]
# 或者: from dbclient.DbClient import getPage [as 别名]
def main():
     
     collectionName = "job_se_10city"
     infoCollectionName = "jobinfo_se_10city"
     
     collectionName = "job_lang_top_corps"
     infoCollectionName = "jobinfo_lang_top_corps"
    
     dbClient = DbClient('localhost', 27017, "jobaly")
     collection = dbClient.getCollection(collectionName)
     infoCollection = dbClient.getCollection(infoCollectionName)
     
     pageSize = 20 
     pageNo = 1
     has_more = True
     pageNum = 10000
     find_sort = None
     find_spec=None

     threadNum = 20
     queue = Queue.Queue()
     for i in range(threadNum):
        t = JobGetter(queue,infoCollection)
        t.setDaemon(True)
        t.start()     
     
     while has_more and pageNo <= pageNum :
        page = dbClient.getPage(collection, find_spec,find_sort, pageSize, pageNo)    
        queue.put( (page,pageNo) )       
        pageNo+=1 
        count =  page.count(with_limit_and_skip = True)
     #   print "count=",count
        if ( count < pageSize ) :
            has_more = False
            
     queue.join()   
开发者ID:folagit,项目名称:resumatcher,代码行数:38,代码来源:multiget.py

示例3: __init__

# 需要导入模块: from dbclient import DbClient [as 别名]
# 或者: from dbclient.DbClient import getPage [as 别名]
class DataProcessor:
    def __init__(self):
        self.dbClient = DbClient("localhost", 27017, "SimilarQuestion")

    @staticmethod
    def processQuestion(question):
        a = {}
        a["qid"] = question["_id"]
        a["title"] = question["title"]
        return a

    @staticmethod
    def processLinkedQuestion(question):
        a = {}
        a["qid"] = question["_id"]
        a["title"] = question["title"]
        a["linked"] = []
        for item in question["items"]:
            b = {}
            b["qid"] = item["question_id"]
            b["title"] = item["title"]
            print b
            a["linked"].append(b)
        return a

    @staticmethod
    def processLinkedQuestion2(question):
        a = {}
        a["qid"] = question["_id"]
        a["linked"] = []
        for item in question["items"]:
            a["linked"].append(item["question_id"])
        return a

    @staticmethod
    def processRelatedQuestion(question):
        a = {}
        a["qid"] = question["_id"]
        a["title"] = question["title"]
        a["related"] = []
        for item in question["items"]:
            b = {}
            b["qid"] = item["question_id"]
            b["title"] = item["title"]
            #    print b
            a["related"].append(b)
        return a

    def dumpDataToFile(self, queFun, collection, find_spec, find_sort, fileName, pageNum):
        pageSize = 1000
        pageNo = 1
        has_more = True
        with open(fileName, "w") as the_file:
            # the_file.write('Hello\n')
            while has_more and pageNo <= pageNum:
                page = self.dbClient.getPage(collection, find_spec, find_sort, pageSize, pageNo)
                pageNo += 1
                count = page.count(with_limit_and_skip=True)
                print "count=", count
                if count < pageSize:
                    has_more = False
                for item in page:
                    a = queFun(item)
                    jstr = json.dumps(a) + "\n"
                    the_file.write(jstr)
                print " page %d saved %d lines in file" % (pageNo - 1, count)

    def dumpPythonQuestions(self, pageNum):
        question_coll = self.dbClient.getCollection("question_test")
        fileName = "..\..\data\pyton_questions.txt"
        self.dumpDataToFile(DataProcessor.processQuestion, question_coll, fileName, pageNum)

    def dumpLinkedQuestions(self, pageNum):
        question_coll = self.dbClient.getCollection("question_link_python")
        fileName = "..\..\data\question_link_python.txt"
        find_spec = {"items": {"$exists": True}, "$where": "this.items.length > 5"}
        find_sort = {"items": {"$size": -1}}

        self.dumpDataToFile(DataProcessor.processLinkedQuestion, question_coll, find_spec, find_sort, fileName, pageNum)

    def dumpLinkedQuestions2(self, pageNum):
        question_coll = self.dbClient.getCollection("question_link_python")
        fileName = "..\..\data\python_linked.txt"
        find_spec = {"items": {"$exists": True}, "$where": "this.items.length > 1"}
        find_sort = {"items": {"$size": -1}}
        self.dumpDataToFile(
            DataProcessor.processLinkedQuestion2, question_coll, find_spec, find_sort, fileName, pageNum
        )

    def dumpRelatedQuestions(self, pageNum):
        question_coll = self.dbClient.getCollection("related_python")
        fileName = "..\..\data\question_related_python.txt"
        find_spec = {"items": {"$exists": True}, "$where": "this.items.length > 5"}
        find_sort = None

        self.dumpDataToFile(
            DataProcessor.processRelatedQuestion, question_coll, find_spec, find_sort, fileName, pageNum
        )
开发者ID:jungin,项目名称:SimilarQuestions,代码行数:100,代码来源:dataprocessor.py


注:本文中的dbclient.DbClient.getPage方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。