本文整理汇总了Python中task.Task.getFrontier方法的典型用法代码示例。如果您正苦于以下问题：Python Task.getFrontier方法的具体用法？Python Task.getFrontier怎么用？Python Task.getFrontier使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类task.Task的用法示例。
在下文中一共展示了Task.getFrontier方法的2个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: createChildren
# 需要导入模块: from task import Task [as 别名]
# 或者: from task.Task import getFrontier [as 别名]
def createChildren(self, task, loopEle):
ele = task.crawlerEle
for child in list(ele):
childTask = Task(task.getFrontier())
childTask.setCrawlerEle(child)
childTask.htmlNode = loopEle
childTask.url = task.url
#childTask.nextCrawlerUrl = task.nextCrawlerUrl
childTask.parentId = task.parentId
childTask.parentNode = task.parentNode
childTask.getFrontier().addTask(childTask)示例2: execute
# 需要导入模块: from task import Task [as 别名]
# 或者: from task.Task import getFrontier [as 别名]
def execute(self, task):
ele = task.crawlerEle
paginateXpath = ele.attrib[datanode.PAGINATE_XPATH] if ele.attrib.has_key(datanode.PAGINATE_XPATH) else None
if paginateXpath == None or task.hasPagiNate:
return
try:
htmlNode = task.htmlNode
if None != htmlNode:
loopEles = parseutil.selectNodes(paginateXpath, task.frontier.getNameSpace(), htmlNode)
paginateMaxXpath = ele.attrib[datanode.PAGINATE_MAX_XPATH] if ele.attrib.has_key(
datanode.PAGINATE_MAX_XPATH) else None
paginateMaxRule = ele.attrib[datanode.PAGINATE_MAX_RULE] if ele.attrib.has_key(
datanode.PAGINATE_MAX_RULE) else None
paginateUrlXpath = ele.attrib[datanode.PAGINATE_URL_XPATH] if ele.attrib.has_key(
datanode.PAGINATE_URL_XPATH) else None
paginateUrlRule = ele.attrib[datanode.PAGINATE_URL_RULE] if ele.attrib.has_key(
datanode.PAGINATE_URL_RULE) else None
maxPage = 0
url = ''
for child in loopEles:
if None != paginateMaxXpath and '' != paginateMaxXpath:
maxPage = parseutil.extractValueByXpath(paginateMaxXpath, task.frontier.getNameSpace(), child)
if None != paginateMaxRule and '' != paginateMaxRule:
maxPage = parseutil.extractValueByRule(paginateMaxRule, maxPage)
else:
maxPage = parseutil.extractValueByRule(paginateMaxRule, child.tostring())
if None != paginateUrlXpath and '' != paginateUrlXpath:
url = parseutil.extractValueByXpath(paginateUrlXpath, task.frontier.getNameSpace(), child)
if None != paginateUrlRule and '' != paginateUrlRule:
url = parseutil.extractValueByRule(paginateUrlRule, url)
else:
if None != paginateUrlRule and '' != paginateUrlRule:
url = parseutil.extractValueByRule(paginateUrlRule, child.tostring())
if ('' != maxPage and int(maxPage) > 0) and url != '':
for i in range(2, int(maxPage) + 1, 1):
nextPageUrl = "http://place.qyer.com" + url + str(i)
childTask = Task(task.getFrontier())
childTask.setCrawlerEle(task.getCrawlerEle())
if task.htmlNode is not None:
childTask.htmlNode = task.htmlNode
childTask.nextCrawlerUrl = nextPageUrl
childTask.parentId = task.parentId
childTask.parentNode = task.parentNode
childTask.hasPagiNate = True
childTask.getFrontier().addTask(childTask)
except Exception, e:
print e, "executing PaginateProcess has occurred exception"