Python Scraper.reset方法代码示例

# 需要导入模块: from scraper import Scraper [as 别名]
# 或者: from scraper.Scraper import reset [as 别名]
class ScraperWrapper(threading.Thread):

    def __init__(self,address='localhost',exchange='barkingowl',broadcast_interval=5,DEBUG=False):
        """
        __init__() constructor setups up the message bus, inits the thread, and sets up 
        local status variables.
        """

        threading.Thread.__init__(self)

        self.uid = str(uuid.uuid4())
        self.address = address
        self.exchange = exchange
        self.DEBUG=DEBUG
        self.interval = broadcast_interval

        # create scraper instance
        self.scraper = Scraper(uid=self.uid)
        self.scraping = False
        self.scraper_thread = None

        # stop control
        self.stopped = False

        #setup message bus
        self.respcon = pika.BlockingConnection(pika.ConnectionParameters(
                                                           host=self.address))
        self.respchan = self.respcon.channel()
        self.respchan.exchange_declare(exchange=self.exchange,type='fanout')

        self.reqcon = pika.BlockingConnection(pika.ConnectionParameters(host=address))
        self.reqchan = self.reqcon.channel()
        self.reqchan.exchange_declare(exchange=exchange,type='fanout')
        result = self.reqchan.queue_declare(exclusive=True)
        queue_name = result.method.queue
        self.reqchan.queue_bind(exchange=exchange,queue=queue_name)
        self.reqchan.basic_consume(self._reqcallback,queue=queue_name,no_ack=True)

        # start our anouncement of availiability
        threading.Timer(self.interval, self.broadcast_available).start()

        if self.DEBUG:
            print "Scraper Wrapper INIT complete."

    def run(self):
        """
        run() is called by the threading sub system when ScraperWrapper.start() is called.  This function
        sets up all of the call abcks needed, as well as begins consuming on the message bus. 
        """
        # setup call backs
        self.scraper.set_finished_callback(self.scraper_finished_callback)
        self.scraper.set_started_callback(self.scraper_started_callback)
        self.scraper.set_broadcast_document_callback(self.scraper_broadcast_document_callback)

        # broadcast availability
        self.broadcast_available()
        self.reqchan.start_consuming()

    def stop(self):
        """
        stop() is called to stop consuming on the message bus, and to stop the scraper from running.
        """
        #self.scraper.stop()
        #if self.scraper_thread != None:
        #    self.scraper_thread.stop()
        self.reqchan.stop_consuming()
        self.stopped = True

    def reset_scraper(self):
        """
        resetscraper() calls reset() within the Scraper class.  This resets the state of the scraper.
        This should not be called unless the scraper has been stoped.
        """
        self.scraper.reset()

    def broadcast_available(self):
        """
        broadcastavailable() broadcasts a message to the message bus saying the scraper is available
        to be dispatched a new url to begin scraping.
        """

        # make sure we are not currently scraping
        if self.scraper.status['busy'] == False:

            packet = {
                'available_datetime': str(datetime.datetime.now())
            }
            payload = {
                'command': 'scraper_available',
                'source_id': self.uid,
                'destination_id': 'broadcast',
                'message': packet
            }
            jbody = json.dumps(payload)
            self.respchan.basic_publish(exchange=self.exchange,routing_key='',body=jbody)

        # boadcast our simple status to the bus
        self.broadcast_simple_status()

        #
#.........这里部分代码省略.........

# 需要导入模块: from scraper import Scraper [as 别名]
# 或者: from scraper.Scraper import reset [as 别名]
class ScraperWrapper(object): #threading.Thread):

    def __init__(self,
                 address='localhost',
                 exchange='barkingowl',
                 heartbeat_interval=30,
                 url_parameters=None,
                 broadcast_interval=5,
                 uid=str(uuid.uuid4()),
                 DEBUG=False):

        #threading.Thread.__init__(self,name="ScraperWrapper : %s" % uid)

        self.uid = str(uuid.uuid4())
        self.address = address
        self.exchange = exchange
        self.heartbeat_interval = heartbeat_interval
        self.url_parameters = url_parameters
        
        self.broadcast_interval = broadcast_interval
        self.uid = uid
        self._DEBUG=DEBUG

        print "ScraperWrapper().__init__(): Creating scraper ..."
 
        self.scraper = Scraper(
            DEBUG = self._DEBUG,
        )
        self.scraping = False
        self.scraper_thread = None

        print "ScraperWrapper().__init__(): Scraper Created."

        self.stopped = False

        self.bus_access = BusAccess(
            uid = self.uid,
            address = self.address,
            exchange = self.exchange,
            heartbeat_interval = self.heartbeat_interval,
            url_parameters = self.url_parameters,
            DEBUG = self._DEBUG,
        )
        self.bus_access.set_callback(
            callback = self._reqcallback,
        )

        #threading.Timer(self.interval, self.broadcast_available).start()
        #threading.Timer(self.interval, self.broadcast_simple_status).start()

        #self.broadcast_status()

        #log( "ScraperWrapper.__init__(): Scraper Wrapper INIT complete.", self.DEBUG )

    #def run(self):
    def start(self):
        self.scraper.set_callbacks(
            start_callback = self.scraper_started_callback,
            finished_callback = self.scraper_finished_callback,
            found_doc_callback = self.scraper_broadcast_document_callback,
            new_url_callback = None,
            bandwidth_limit_callback = None,
            memory_limit_callback = None,
            error_callback = None,
        )

        self.broadcast_status()

    def stop(self):
        self.bus_access.stop_listening()
        self.scraper.stop()
        self.stopped = True

    def reset_scraper(self):
        self.scraper.reset()

    def broadcast_status(self):

        if self._DEBUG == True:
            print "ScraperWrapper().broadcast_status(): Entering status loop."
        
        while not self.scraping and not self.stopped:

            if self._DEBUG == True:
                print "ScraperWrapper.broadcast_status() sending status pulse ..."

            if self.scraping == False and self.scraper._data['working'] == False:
                packet = {
                    'available_datetime': str(datetime.datetime.now())
                }
                self.bus_access.send_message(
                    command = 'scraper_available',
                    destination_id = 'broadcast',
                    message = packet,
                )

            '''
            packet = {
                'working': self.scraper._data['working'],
                'seen_url_count': len(self.scraper._data['seen_urls']),
#.........这里部分代码省略.........