Python Scheduler.shutdown方法代码示例

# 需要导入模块: from scheduler import Scheduler [as 别名]
# 或者: from scheduler.Scheduler import shutdown [as 别名]
def main():
    config = ConfigParser()
    config.read(expanduser("~/.wn.conf"))
    wndb = WNDB(path=expanduser(config['Database']['path']))
    interpreter = Interpreter()
    telegram = Telegram(config, interpreter)
    scheduler = Scheduler(wndb, telegram)

    thread = Thread(target=scheduler.run)

    thread.start()

    while True:
        try:
            sleep(.1)
        except (KeyboardInterrupt, SystemExit):
            break

    scheduler.shutdown()
    thread.join()
    telegram.shutdown()
    interpreter.shutdown()

# 需要导入模块: from scheduler import Scheduler [as 别名]
# 或者: from scheduler.Scheduler import shutdown [as 别名]
class CrawlJob(object):

    def __init__(self, hq, jobname):
        self.hq = hq
        self.jobconfigs = self.hq.jobconfigs
        self.jobname = jobname
        self.mapper = CrawlMapper(self, hqconfig.NWORKSETS_BITS)
        self.scheduler = Scheduler(hqconfig.worksetdir(self.jobname),
                                   self.mapper)

        self.inq = PooledIncomingQueue(
            qdir=hqconfig.inqdir(self.jobname),
            buffsize=1000)

        self._dispatcher_mode = hqconfig.get(
            ('jobs', self.jobname, 'dispatcher'), 'internal')
                                            
        self.dispatcher = None
        #self.init_dispatcher()

        # currently disabled by default - too slow
        self.use_crawlinfo = False
        self.save_crawlinfo = False

        self.last_inq_count = 0

    PARAMS = [('use_crawlinfo', bool),
              ('save_crawlinfo', bool),
              ('dispatcher_mode', str)]

    @property
    def dispatcher_mode(self):
        return self._dispatcher_mode
    @dispatcher_mode.setter
    def dispatcher_mode(self, value):
        self._dispatcher_mode = value
        if value == 'external':
            self.shutdown_dispatcher()

    def init_dispatcher(self):
        if self.dispatcher: return
        if self.dispatcher_mode == 'external':
            raise RuntimeError, 'dispatcher mode is %s' % self.dispatcher_mode
        self.dispatcher = Dispatcher(self.hq.get_domaininfo(),
                                     self.jobname,
                                     mapper=self.mapper,
                                     scheduler=self.scheduler,
                                     inq=self.inq.rqfile)
        
    def shutdown_dispatcher(self):
        if not self.dispatcher: return
        logging.info("shutting down dispatcher")
        self.dispatcher.shutdown()
        self.dispatcher = None

    def shutdown(self):
        logging.info("shutting down scheduler")
        self.scheduler.shutdown()
        logging.info("closing incoming queues")
        self.inq.flush()
        self.inq.close()
        self.shutdown_dispatcher()
        logging.info("done.")

    def get_status(self):
        r = dict(job=self.jobname, oid=id(self))
        r['sch'] = self.scheduler and self.scheduler.get_status()
        r['inq'] = self.inq and self.inq.get_status()
        return r

    def get_workset_status(self):
        r = dict(job=self.jobname, crawljob=id(self))
        if self.scheduler:
            r['sch'] = id(self.scheduler)
            r['worksets'] = self.scheduler.get_workset_status()
        return r
        
    def workset_activating(self, *args):
        self.init_dispatcher()
        self.dispatcher.workset_activating(*args)

    def schedule(self, curis):
        '''schedule curis bypassing seen-check. typically used for starting
           new crawl cycle.'''
        scheduled = 0
        for curi in curis:
            self.scheduler.schedule(curi)
            scheduled += 1
        return dict(processed=scheduled, scheduled=scheduled)

    def discovered(self, curis):
        return self.inq.add(curis)

    def processinq(self, maxn):
        self.init_dispatcher()
        return self.dispatcher.processinq(maxn)

    def makecuri(self, o):
        # temporary rescue measure. delete after everything's got fixed.
        a = o.get('a')
#.........这里部分代码省略.........

# 需要导入模块: from scheduler import Scheduler [as 别名]
# 或者: from scheduler.Scheduler import shutdown [as 别名]
class DBPictureBox(PictureBox):
   '''
   This class extends a regular .NET PictureBox so that it's image can be set
   asynchronously by giving it a SeriesRef or IssueRef object (the 
   set_image_ref method).  This ref object is then used to query the 
   comic book database to find the appropriate image (if any) to set.   

   Because accessing a database can be slow and potentially prone to errors, 
   this class contains a number of features to automatically handle
   problems that may occur:

   1) All image retrieval happens asyncronously on an external thread.   The 
   final result is only loaded into this picturebox when the image retrieval is
   finished.  If many image retrievals are requested nearly simulataneously, 
   most of them will be ignored, and only the most recent one is guaranteed to 
   actually be performed (and update the displayed image.)

   2) All image retrieval is short-term cached, so if you switch the image ref 
   back to a previous one, it will automatically use the cached image instead of 
   reloading.
   
   '''


   #===========================================================================
   def __init__(self): 
      ''' Defines member variables for new instances of this class. '''
      # the ref of whatever image should currently be displayed, or None
      self.__current_image_ref = None
      
      # a simple "last-in-and-ignore-everything-else" scheduler
      self.__scheduler = Scheduler()
      
      # our cache of loaded image objects. {(imageref,issuehint)->Image}
      self.__image_cache = {}
      
      # the image that gets displayed if we have nothing else to display
      self.__unknown_image = Resources.createComicVineLogo()
      
      # the image that gets displayed while we are loading another image
      self.__loading_image = self.__copy_transparent(self.__unknown_image)
      
      self._initialize()


   #===========================================================================
   def _initialize(self):
      ''' Initial configuration for new instances of this class '''
      
      PictureBox.__init__(self)      
      def visibility_changed(sender, args):
         if self.Visible: self.__update_image()
         
      self.VisibleChanged += visibility_changed
      self.SizeMode = PictureBoxSizeMode.StretchImage
      self.set_image_ref(None)


   #===========================================================================
   def free(self):
      ''' Explicitly frees all resources held by this object. '''
      self.__scheduler.shutdown(True) # blocks; safer even if gui locks a little
      self.__unknown_image.Dispose()
      self.__loading_image.Dispose()
      for image in self.__image_cache.values():
         image.Dispose()
      self.__image_cache = {}
      PictureBox.Dispose(self, True)
      

   #===========================================================================
   def set_image_ref(self, ref):
      '''
      Sets the image displayed in this picturebox to whatever image
      corresponds to the given ref (a SeriesRef or IssueRef) in the database.
      If the given value is None, or unavailable, a placeholder image gets used.
      '''
      
      self.__current_image_ref = ref
      if self.Visible:
         self.__update_image()
     
    
   #===========================================================================
   def __copy_transparent(self, image):
      ''' Creates a semi-transparent copy of the given image ''' 
      
      b = Bitmap( image.Width, image.Height )
      g = Graphics.FromImage(b)
      cm = ColorMatrix()
      cm.Matrix33 = 0.3
      ia = ImageAttributes()
      ia.SetColorMatrix(cm)
      g.DrawImage(image, Rectangle(0,0, image.Width, image.Height), 0,0,\
         image.Width, image.Height, GraphicsUnit.Pixel, ia)
      return b

   #===========================================================================
   def __update_image(self):
      '''
#.........这里部分代码省略.........

# 需要导入模块: from scheduler import Scheduler [as 别名]
# 或者: from scheduler.Scheduler import shutdown [as 别名]
class CrawlJob(object):
    NWORKSETS_BITS = 8

    def __init__(self, jobconfigs, jobname, crawlinfo, domaininfo):
        self.jobconfigs = jobconfigs
        self.jobname = jobname
        self.mapper = CrawlMapper(self, self.NWORKSETS_BITS)
        self.workset_state = [0 for i in range(self.mapper.nworksets)]

        # seen-db initialization is delayed until it's actually needed
        self.seen = None
        #self.seen = Seen(dbdir=os.path.join(HQ_HOME, 'seen', self.jobname))
        self.crawlinfodb = crawlinfo
        self.domaininfo = domaininfo
        self.scheduler = Scheduler(hqconfig.worksetdir(self.jobname),
                                   self.mapper)
        # self.inq = HashSplitIncomingQueue(
        #     qdir=hqconfig.inqdir(self.jobname),
        #     buffsize=500)
        self.inq = PooledIncomingQueue(
            qdir=hqconfig.inqdir(self.jobname),
            buffsize=1000)

        self.diverter = Diverter(self.jobname, self.mapper)

        #self.discovered_executor = ThreadPoolExecutor(poolsize=1)

        # currently disabled by default - too slow
        self.use_crawlinfo = False
        self.save_crawlinfo = False

    PARAMS = [('use_crawlinfo', bool),
              ('save_crawlinfo', bool)]

    def shutdown(self):
        logging.info("shutting down scheduler")
        self.scheduler.shutdown()
        logging.info("shutting down diverter")
        self.diverter.shutdown()
        if self.seen:
            logging.info("closing seen db")
            self.seen.close()
        logging.info("closing incoming queues")
        self.inq.flush()
        self.inq.close()
        logging.info("shutting down crawlinfo")
        self.crawlinfodb.shutdown()
        logging.info("done.")
        #self.discovered_executor.shutdown()

    def get_status(self):
        r = dict(job=self.jobname, oid=id(self))
        r['seen'] = self.seen and self.seen.get_status()
        r['sch'] = self.scheduler and self.scheduler.get_status()
        r['inq'] = self.inq and self.inq.get_status()
        return r

    def get_workset_status(self):
        r = dict(job=self.jobname, crawljob=id(self))
        if self.scheduler:
            r['sch'] = id(self.scheduler)
            r['worksets'] = self.scheduler.get_workset_status()
        return r
        
    #def discovered_async(self, curis):
    #    return self.inq.add(curis)

    def get_domaininfo(self, url):
        uc = urlsplit(url)
        host = uc.netloc
        p = host.find(':')
        if p > 0: host = host[:p]
        di = self.domaininfo.get(host)
        return di
        
    def schedule(self, curis):
        '''schedule curis bypassing seen-check. typically used for starting
           new crawl cycle.'''
        scheduled = 0
        for curi in curis:
            self.scheduler.schedule(curi)
            scheduled += 1
        return dict(processed=scheduled, scheduled=scheduled)

    def discovered(self, curis):
        return self.inq.add(curis)
        
    def is_client_active(self, clid):
        """is client clid active?"""
        # TODO: update ZooKeeper when active status changes
        #t = self.client_last_active.get(str(clid))
        return self.scheduler.is_active(clid)

    def is_workset_active(self, wsid):
        """is workset wsid assigned to any active client?"""
        clid = self.mapper.worksetclient[wsid]
        return self.is_client_active(clid)

    def workset_activating(self, wsid):
        """activates working set wsid; start sending CURIs to Scheduler
#.........这里部分代码省略.........

# 需要导入模块: from scheduler import Scheduler [as 别名]
# 或者: from scheduler.Scheduler import shutdown [as 别名]
import json
import uuid
from scheduler import Scheduler

if __name__ == '__main__':
    from kazoo.client import KazooClient
    import sys
    task = {
        'job_id': 'test'
    }
    task_id = uuid.uuid4().hex
    print(task_id)
    target = 'test'
    zk = KazooClient()
    zk.start()
    zk.ensure_path('/msched/agents/{0}/tasks'.format(target))
    zk.ensure_path('/msched/tasks/{0}'.format(task_id))
    zk.set('/msched/tasks/{0}'.format(task_id), json.dumps(task).encode())
    zk.ensure_path('/msched/tasks/{0}/targets/{1}'.format(task_id, target))
    zk.set('/msched/tasks/{0}/targets/{1}'.format(task_id, target), b'N')
    zk.ensure_path('/msched/signal/{0}'.format(task_id))
    if len(sys.argv) >= 1:
        sys.exit()
    sched = Scheduler(zk, 'msched')
    sched.watch()
    try:
        sched.join()
    except KeyboardInterrupt:
        sched.shutdown()

# 需要导入模块: from scheduler import Scheduler [as 别名]
# 或者: from scheduler.Scheduler import shutdown [as 别名]
import os
from tornado.options import options, define, parse_command_line, parse_config_file
from scheduler import Scheduler

define('hosts', default='127.0.0.1:2181', type=str, help='zookeeper hosts')
define('root', default='/msched', type=str, help='zookeeper root node')

if __name__ == '__main__':
    if os.path.exists('/etc/msched/scheduler.conf'):
        parse_config_file('/etc/msched/scheduler.conf')
    if os.path.exists('./application.conf'):
        parse_config_file('./application.conf')
    parse_command_line()
    scheduler = Scheduler(hosts=options.hosts, root=options.root)
    scheduler.start()
    try:
        scheduler.join()
    except KeyboardInterrupt:
        scheduler.shutdown()

# 需要导入模块: from scheduler import Scheduler [as 别名]
# 或者: from scheduler.Scheduler import shutdown [as 别名]
class InqueueProcessor(object):
    inqwatcher = None
    # TODO: allow different type of Dispatcher for each job.
    def __init__(self, job, dispatcher_type, maxn):
        self.job = job
        self.maxn = maxn

        self.domaininfo = hqconfig.factory.domaininfo()
        self.jobconfigs = hqconfig.factory.jobconfigs()
        self.coordinator = hqconfig.factory.coordinator()

        # per-job objects
        # TODO: process multiple jobs in one process
        self.mapper = CrawlMapper(CrawlJob(self.job, self.jobconfigs),
                                  hqconfig.NWORKSETS_BITS)
        self.scheduler = Scheduler(hqconfig.worksetdir(self.job),
                                   self.mapper, reading=False)
        self.inqueue = IncomingQueue(hqconfig.inqdir(self.job),
                                     deq=PriorityDequeue)
        self.dispatcher = build_dispatcher(dispatcher_type,
                                           self.domaininfo, self.job,
                                           mapper=self.mapper,
                                           scheduler=self.scheduler,
                                           inqueue=self.inqueue)

        if os.uname()[0] == 'Linux':
            if InqueueProcessor.inqwatcher is None:
                iqw = InqueueProcessor.inqwatcher = InqueueWatcher()
                iqw.start()
            self.watch = InqueueProcessor.inqwatcher.addwatch(self.inqueue.qdir)

    def shutdown(self):
        self.dispatcher.shutdown()
        self.scheduler.shutdown()
        self.jobconfigs.shutdown()
        self.domaininfo.shutdown()

    def stop(self):
        self.__repeat = False

    def processinq(self):
        # t and job should be set in Dispatcher.processinq
        start = time.time()
        r = self.dispatcher.processinq(self.maxn)
        r['t'] = time.time() - start
        r['job'] = self.job
        r.update(
            ps=r.get('processed', 0)/r['t'] if r['t'] != 0 else 0,
            eps=r.get('scheduled', 0)/r['t'] if r['t'] != 0 else 0
            )
        return r

    def run(self, repeat=True, exhaust_delay=60):
        self.__repeat = repeat
        do_flush = False
        while 1:
            r = self.processinq()
            logging.info("{job} {scheduled:d}/{processed:d} "
                         "X:{excluded:d} T:{t:.3f}(D{td:.3f},S{ts:.3f}) "
                         "{eps:8.2f}/{ps:8.2f}/s".format(**r))
            if not r.get('processed'):
                # flush worksets before sleeping - this is not perfect,
                # but better-than-nothing solution until we implement
                # inter-process communication.
                if do_flush:
                    self.dispatcher.flush_worksets()
                    do_flush = False
                logging.info("inq exhausted, sleeping for %ds",
                             exhaust_delay)
                # because wait_available cannot be interrupted by INT signal,
                # we repeat short waits for exhaust_delay.
                #time.sleep(exhaust_delay)
                until = time.time() + exhaust_delay
                while time.time() < until and self.__repeat:
                    if self.watch.wait(1.0):
                        break
            else:
                # flush if clients are starved.
                if self.scheduler.flush_starved():
                    logging.info("workset flushed")
                    do_flush = False
                else:
                    do_flush = (r.get('scheduled', 0) > 0)
            if not self.__repeat:
                break

# 需要导入模块: from scheduler import Scheduler [as 别名]
# 或者: from scheduler.Scheduler import shutdown [as 别名]

#.........这里部分代码省略.........
   
   # ==========================================================================
   def __build_prevbutton(self):
      ''' Builds and returns the 'previous' button for this panel. '''

      button = Button()
      button.Location = Point(2, 332)
      button.Size = Size(20, 24)
      button.Text = '<'
      button.Font = Font(button.Font, FontStyle.Bold)
      button.UseVisualStyleBackColor = True
      button.Click += self.__button_click_fired
      # note: this button's visibility is manipulated by __update
      return button

      
   # ==========================================================================
   def free(self): 
      ''' 
      Free all resources allocated by this class when it is no longer needed.
      '''
      
      # sets the alternate issue cover that the user may have chosen
      # see get_alt_issue_cover_choice() for more details
      if type(self.__ref) == IssueRef:
         issue_ref = self.__ref
         button_model = self.__button_cache[issue_ref]
         if button_model:
            if button_model.can_decrement():
               image_ref = button_model.get_current_ref()
               if utils.is_string(image_ref):
                  self.__alt_cover_choice = (issue_ref, image_ref)
      
      self.__finder_scheduler.shutdown(False)
      self.__setter_scheduler.shutdown(False)
      self.set_ref(None)
      self.__coverpanel.free()
      self.__prevbutton = None
      self.__nextbutton = None
      self.__label = None
      self.Dispose()


   # ==========================================================================
   def set_ref(self, ref):
      '''
      Sets the comic IssueRef or SeriesRef that this panel is displaying.
      'ref'-> the IssueRef or SeriesRef that we are displaying, or None.
      '''

      run_in_background = type(ref) == SeriesRef and self.__issue_num_hint_s       
      if run_in_background:
         # 1a. our ref is a SeriesRef.  use our issue num hint to try to convert
         #     it into an IssueRef.  use a simple cache to avoid requerying
         def maybe_convert_seriesref_to_issue_ref(ref):
            if not ref in self.__series_cache:
               issue_ref = db.query_issue_ref(ref, self.__issue_num_hint_s)
               self.__series_cache[ref] = issue_ref if issue_ref else ref
                  
            # 1b. go back to the application thread to do the actual ref change
            def change_ref():  
               self.__ref = self.__series_cache[ref]
               self.__update()
            utils.invoke(self.__coverpanel, change_ref, True)
            
         def dummy(): # I don't know why this is needed 

# 需要导入模块: from scheduler import Scheduler [as 别名]
# 或者: from scheduler.Scheduler import shutdown [as 别名]
class CrawlJob(object):
    def __init__(self, jobconfigs, jobname, domaininfo):
        self.jobconfigs = jobconfigs
        self.jobname = jobname
        self.qdir = hqconfig.inqdir(self.jobname)

        self.inq = FileDequeue(self.qdir, reader=FPSortingQueueFileReader)

        self.mapper = WorksetMapper(hqconfig.NWORKSETS_BITS)
        self.seen = Seen(dbdir=hqconfig.seendir(self.jobname))
        self.domaininfo = domaininfo
        
        self.scheduler = Scheduler(self.jobname, self.mapper)

    def shutdown(self):
        logging.info("closing seen db")
        self.seen.close()
        logging.info("shutting down scheduler")
        self.scheduler.shutdown()
        logging.info("done.")

    def get_status(self):
        r = dict(job=self.jobname, oid=id(self))
        r['sch'] = self.scheduler and self.scheduler.get_status()
        r['inq'] = self.inq and self.inq.get_status()
        return r

    # def get_workset_status(self):
    #     r = dict(job=self.jobname, crawljob=id(self))
    #     if self.scheduler:
    #         r['sch'] = id(self.scheduler)
    #         r['worksets'] = self.scheduler.get_workset_status()
    #     return r
        
    def get_domaininfo(self, url):
        uc = urlsplit(url)
        host = uc.netloc
        p = host.find(':')
        if p > 0: host = host[:p]
        di = self.domaininfo.get(host)
        return di
        
    def schedule(self, curis):
        '''schedule curis bypassing seen-check. typically used for starting
           new crawl cycle.'''
        scheduled = 0
        for curi in curis:
            #self.scheduler.schedule(curi)
            ws = self.mapper.workset(curi)
            self.worksets[ws].schedule(curi)
            scheduled += 1
        return dict(processed=scheduled, scheduled=scheduled)

    def processinq(self, maxn):
        '''process incoming queue. maxn paramter adivces
        upper limit on number of URIs processed in this single call.
        actual number of URIs processed may exceed it if incoming queue
        stores URIs in chunks.'''
        result = dict(processed=0, scheduled=0, excluded=0, td=0.0, ts=0.0)
        for count in xrange(maxn):
            t0 = time.time()
            furi = self.inq.get(0.01)
            
            result['td'] += (time.time() - t0)
            if furi is None: break
            result['processed'] += 1
            di = self.get_domaininfo(furi['u'])
            if di and di['exclude']:
                result['excluded'] += 1
                continue
            t0 = time.time()
            suri = self.seen.already_seen(furi)
            if suri['e'] < int(time.time()):
                if 'w' in furi:
                    w = furi['w']
                else:
                    w = dict()
                    for k in ('p','v','x'):
                        m = furi.get(k)
                        if m is not None:
                            w[k] = m
                curi = dict(u=furi['u'], id=suri['_id'], a=w)
                self.scheduler.schedule(curi)
                result['scheduled'] += 1
            result['ts'] += (time.time() - t0)
        # currently no access to MongoDB
        #self.mongo.end_request()
        return result

    def makecuri(self, o):
        return o

    def flush(self):
        self.seen.flush()
        return self.scheduler.flush()