本文整理汇总了Python中scraper.Scraper.setFinishedCallback方法的典型用法代码示例。如果您正苦于以下问题：Python Scraper.setFinishedCallback方法的具体用法？Python Scraper.setFinishedCallback怎么用？Python Scraper.setFinishedCallback使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类scraper.Scraper的用法示例。
在下文中一共展示了Scraper.setFinishedCallback方法的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: ScraperWrapper
# 需要导入模块: from scraper import Scraper [as 别名]
# 或者: from scraper.Scraper import setFinishedCallback [as 别名]
class ScraperWrapper(threading.Thread):
def __init__(self,address='localhost',exchange='barkingowl',DEBUG=False):
threading.Thread.__init__(self)
self.uid = str(uuid.uuid4())
self.address = address
self.exchange = exchange
self.DEBUG=DEBUG
self.interval = 1
# create scraper instance
self.scraper = Scraper(uid=self.uid,DEBUG=DEBUG)
self.scraping = False
#setup message bus
self.respcon = pika.BlockingConnection(pika.ConnectionParameters(
host=self.address))
self.respchan = self.respcon.channel()
self.respchan.exchange_declare(exchange=self.exchange,type='fanout')
self.reqcon = pika.BlockingConnection(pika.ConnectionParameters(host=address))
self.reqchan = self.reqcon.channel()
self.reqchan.exchange_declare(exchange=exchange,type='fanout')
result = self.reqchan.queue_declare(exclusive=True)
queue_name = result.method.queue
self.reqchan.queue_bind(exchange=exchange,queue=queue_name)
self.reqchan.basic_consume(self.reqcallback,queue=queue_name,no_ack=True)
if self.DEBUG:
print "Scraper Wrapper INIT complete."
def run(self):
# setup call backs
self.scraper.setFinishedCallback(self.scraperFinishedCallback)
self.scraper.setStartedCallback(self.scraperStartedCallback)
self.scraper.setBroadcastDocCallback(self.scraperBroadcastDocCallback)
# broadcast availability
self.broadcastavailable()
self.reqchan.start_consuming()
def stop(self):
self.scraper.stop()
self.reqchan.stop_consuming()
def broadcastavailable(self):
if self.scraper.status['busy'] == True:
# we are currently scraping, so we are not available - don't broadcast
return
isodatetime = strftime("%Y-%m-%d %H:%M:%S")
packet = {
'availabledatetime': str(isodatetime)
}
payload = {
'command': 'scraper_available',
'sourceid': self.uid,
'destinationid': 'broadcast',
'message': packet
}
jbody = simplejson.dumps(payload)
self.respchan.basic_publish(exchange=self.exchange,routing_key='',body=jbody)
#
# TODO: move this over to it's own timer, no need to do it here.
#
if self.scraper.stopped():
raise Exception("Scraper Wrapper Exiting")
else:
threading.Timer(self.interval, self.broadcastavailable).start()
def broadcaststatus(self):
isodatetime = strftime("%Y-%m-%d %H:%M:%S")
packet = {
'status': self.scraper.status,
'urldata': self.status['urldata'],
'statusdatetime': str(isodatetime)
}
payload = {
'command': 'scraper_status',
'sourceid': self.uid,
'destinationid': 'broadcast',
'message': packet
}
jbody = simplejson.dumps(payload)
#time.sleep(.5)
self.respchan.basic_publish(exchange=self.exchange,routing_key='',body=jbody)
def broadcastsimplestatus(self):
isodatetime = strftime("%Y-%m-%d %H:%M:%S")
if self.scraper.status['urldata'] == {}:
targeturl = 'null'
else:
targeturl = self.scraper.status['urldata']['targeturl']
packet = {
'busy': self.scraper.status['busy'],
'linkcount': self.scraper.status['linkcount'],
#.........这里部分代码省略.........