Python LifoQueue.task_done方法代码示例

# 需要导入模块: from Queue import LifoQueue [as 别名]
# 或者: from Queue.LifoQueue import task_done [as 别名]

#.........这里部分代码省略.........
    def aliveThreads( self ):
        '''Return a map of thread identifiers and their alive status.

        All threads are alive until notifyThreadsToStop() is
        called. After that, they start dying. Call joinThreads() to wait
        for the last thread to die.
        
        '''
        at = {}
        for thread in self._dirtyLayerThreads:
            if thread.ident:
                at[thread.ident] = thread.isAlive()
        return at

    def _dirtyLayersWorker( self ):
        while self._keepRendering:
            try:
                ims, tile_nr, stack_id, image_req, timestamp, cache = self._dirtyLayerQueue.get(True, self.THREAD_HEARTBEAT)
            except (Empty, TypeError):
                #the TypeError occurs when the self._dirtyLayerQueue
                #is already None when the thread is being shut down
                #on program exit.
                #This avoids a lot of warnings.
                continue
            try:
                if timestamp > cache.layerTimestamp( stack_id, ims, tile_nr ):
                    img = image_req.wait()
                    cache.updateTileIfNecessary( stack_id, ims, tile_nr, timestamp, img )
                    if stack_id == self._current_stack_id and cache is self._cache:
                        self.changed.emit(QRectF(self.tiling.imageRects[tile_nr]))
            except KeyError:
                pass
            finally:
                self._dirtyLayerQueue.task_done()

    def _refreshTile( self, stack_id, tile_no ):
        try:
            if self._cache.tileDirty( stack_id, tile_no ):
                self._cache.setTileDirty(stack_id, tile_no, False)
                img = self._renderTile( stack_id, tile_no )
                self._cache.setTile( stack_id, tile_no, img, self._sims.viewVisible(), self._sims.viewOccluded() )

                # refresh dirty layer tiles 
                for ims in self._sims.viewImageSources():
                    if self._cache.layerDirty(stack_id, ims, tile_no) and not self._sims.isOccluded(ims) and self._sims.isVisible(ims):
                        req = (ims,
                               tile_no,
                               stack_id,
                               ims.request(self.tiling.imageRects[tile_no]),
                               time.time(),
                               self._cache)
                        try:
                            self._dirtyLayerQueue.put_nowait( req )
                        except Full:
                            warnings.warn("Request queue full. Dropping tile refresh request. Increase queue size!")
        except KeyError:
            pass

    def _renderTile( self, stack_id, tile_nr ):
        qimg = QImage(self.tiling.imageRects[tile_nr].size(), QImage.Format_ARGB32_Premultiplied)
        qimg.fill(Qt.white)

        p = QPainter(qimg)
        for i, v in enumerate(reversed(self._sims)):
            visible, layerOpacity, layerImageSource = v
            if not visible:

# 需要导入模块: from Queue import LifoQueue [as 别名]
# 或者: from Queue.LifoQueue import task_done [as 别名]
class SaveManager(QObject):

    start_save = pyqtSignal()
    report_error = pyqtSignal(object)
    save_done = pyqtSignal()

    def __init__(self, parent):
        QObject.__init__(self, parent)
        self.count = 0
        self.last_saved = -1
        self.requests = LifoQueue()
        t = Thread(name='save-thread', target=self.run)
        t.daemon = True
        t.start()
        self.status_widget = w = SaveWidget(parent)
        self.start_save.connect(w.start, type=Qt.QueuedConnection)
        self.save_done.connect(w.stop, type=Qt.QueuedConnection)

    def schedule(self, tdir, container):
        self.count += 1
        self.requests.put((self.count, tdir, container))

    def run(self):
        while True:
            x = self.requests.get()
            if x is None:
                self.requests.task_done()
                self.__empty_queue()
                break
            try:
                count, tdir, container = x
                self.process_save(count, tdir, container)
            except:
                import traceback
                traceback.print_exc()
            finally:
                self.requests.task_done()

    def __empty_queue(self):
        ' Only to be used during shutdown '
        while True:
            try:
                self.requests.get_nowait()
            except Empty:
                break
            else:
                self.requests.task_done()

    def process_save(self, count, tdir, container):
        if count <= self.last_saved:
            shutil.rmtree(tdir, ignore_errors=True)
            return
        self.last_saved = count
        self.start_save.emit()
        try:
            self.do_save(tdir, container)
        except:
            import traceback
            self.report_error.emit(traceback.format_exc())
        self.save_done.emit()

    def do_save(self, tdir, container):
        try:
            save_container(container, container.path_to_ebook)
        finally:
            shutil.rmtree(tdir, ignore_errors=True)

    @property
    def has_tasks(self):
        return bool(self.requests.unfinished_tasks)

    def wait(self, timeout=30):
        if timeout is None:
            self.requests.join()
        else:
            try:
                join_with_timeout(self.requests, timeout)
            except RuntimeError:
                return False
        return True

    def shutdown(self):
        self.requests.put(None)

# 需要导入模块: from Queue import LifoQueue [as 别名]
# 或者: from Queue.LifoQueue import task_done [as 别名]
class ThreadPool(object):
    def __init__(self, threadNum, max_tasks_per_period=10, seconds_per_period=30):
        self.pool = []  # 线程池
        self.threadNum = threadNum  # 线程数
        self.runningLock = Lock()  # 线程锁
        self.taskLock = Lock()  # getTask函数的锁
        self.running = 0  # 正在run的线程数

        # 设置为LIFO队列：在抓取了第一个post的页面后，随后需要添加所有其后的评论页，
        # 使用LIFO队列可以保证尽快将第一个post的所有评论抓取到，并存储
        self.taskQueue = LifoQueue()  # 任务队列

        # 一分钟内允许的最大访问次数
        self.max_tasks_per_period = max_tasks_per_period
        # 定制每分钟含有的秒数
        self.seconds_per_period = seconds_per_period
        # 当前周期内已经访问的网页数量
        self.currentPeriodVisits = 0
        # 将一分钟当作一个访问周期，记录当前周期的开始时间
        self.periodStart = time.time()  # 使用当前时间初始化

    def startThreads(self):
        """Create a certain number of threads and started to run 
        All Workers share the same ThreadPool
        """
        # 开始当前的抓取周期
        self.periodStart = time.time()
        for i in range(self.threadNum):
            self.pool.append(Worker(self, i))

    def stopThreads(self):
        for thread in self.pool:
            thread.stop()
            thread.join()
        del self.pool[:]

    def putTask(self, func, *args, **kargs):
        self.taskQueue.put((func, args, kargs))

    def getTask(self, *args, **kargs):
        # 进行访问控制: 判断当前周期内访问的网页数目是否大于最大数目
        if self.currentPeriodVisits >= self.max_tasks_per_period - 2:
            timeNow = time.time()
            seconds = timeNow - self.periodStart
            if seconds < self.seconds_per_period:  # 如果当前还没有过一分钟,则sleep
                remain = self.seconds_per_period - seconds
                print "ThreadPool Waiting for " + str(remain) + " seconds."
                time.sleep(int(remain + 1))

            self.periodStart = time.time()  # 重新设置开始时间
            self.currentPeriodVisits = 0

        try:
            # task = self.taskQueue.get(*args, **kargs)
            task = self.taskQueue.get_nowait()
        except Empty:
            return (None, None, None)

        self.currentPeriodVisits += 1

        return task

    def taskJoin(self, *args, **kargs):
        """Queue.join: Blocks until all items in the queue have been gotten and processed.
        """
        self.taskQueue.join()

    def taskDone(self, *args, **kargs):
        self.taskQueue.task_done()

    def increaseRunsNum(self):
        self.runningLock.acquire()
        self.running += 1  # 正在运行的线程数加1
        self.runningLock.release()

    def decreaseRunsNum(self):
        self.runningLock.acquire()
        self.running -= 1
        self.runningLock.release()

    def getTaskLeft(self):
        # 线程池的所有任务包括：
        # taskQueue中未被下载的任务, resultQueue中完成了但是还没被取出的任务, 正在运行的任务
        # 因此任务总数为三者之和
        return self.taskQueue.qsize() + self.running

# 需要导入模块: from Queue import LifoQueue [as 别名]
# 或者: from Queue.LifoQueue import task_done [as 别名]
class Imagery:

    # Number of simultaneous connections to imagery provider and concurrent placement layouts.
    # Limit to 1 to prevent worker threads slowing UI too much.
    connections=1	

    def __init__(self, canvas):

        self.providers={'Bing': self.bing_setup, 'ArcGIS': self.arcgis_setup, 'MapQuest': self.mq_setup }

        self.canvas=canvas
        self.imageryprovider=None
        self.provider_base=None
        self.provider_url=None
        self.provider_logo=None	# (filename, width, height)
        self.provider_levelmin=self.provider_levelmax=0
        self.placementcache={}	# previously created placements (or None if image couldn't be loaded), indexed by quadkey.
                                # placement may not be laid out if image is still being fetched.
        self.tile=(0,999)	# X-Plane 1x1degree tile - [lat,lon] of SW
        self.loc=None
        self.dist=0

        self.filecache=Filecache()

        # Setup a pool of worker threads
        self.workers=[]
        self.q=LifoQueue()
        for i in range(Imagery.connections):
            t=threading.Thread(target=self.worker)
            t.daemon=True	# this doesn't appear to work for threads blocked on Queue
            t.start()
            self.workers.append(t)

    # Worker thread
    def worker(self):
        # Each thread gets its own tessellators in thread-local storage
        tls=threading.local()
        tls.tess=gluNewTess()
        gluTessNormal(tls.tess, 0, -1, 0)
        gluTessProperty(tls.tess, GLU_TESS_WINDING_RULE, GLU_TESS_WINDING_NEGATIVE)
        gluTessCallback(tls.tess, GLU_TESS_VERTEX_DATA,  ElevationMeshBase.tessvertex)
        gluTessCallback(tls.tess, GLU_TESS_EDGE_FLAG,    ElevationMeshBase.tessedge)
        tls.csgt=gluNewTess()
        gluTessNormal(tls.csgt, 0, -1, 0)
        gluTessProperty(tls.csgt, GLU_TESS_WINDING_RULE, GLU_TESS_WINDING_ABS_GEQ_TWO)
        gluTessCallback(tls.csgt, GLU_TESS_VERTEX_DATA,  ElevationMeshBase.tessvertex)
        gluTessCallback(tls.csgt, GLU_TESS_COMBINE,      ElevationMeshBase.tesscombinetris)
        gluTessCallback(tls.csgt, GLU_TESS_EDGE_FLAG,    ElevationMeshBase.tessedge)

        while True:
            (fn, args)=self.q.get()
            if not fn: exit()	# Die!
            fn(tls, *args)
            self.q.task_done()


    def exit(self):
        # Closing down
        self.filecache.writedir()
	# kill workers
        for i in range(Imagery.connections):
            self.q.put((None, ()))	# Top priority! Don't want to hold up program exit
        # wait for them
        for t in self.workers:
            t.join()


    def reset(self):
        # Called on reload or on new tile. Empty the cache and forget allocations since:
        # a) in the case of reload, textures have been dropped and so would need to be reloaded anyway;
        # b) try to limit cluttering up the VBO with allocations we may not need again;
        # c) images by straddle multiple tiles and these would need to be recalculated anyway.
        for placement in self.placementcache.itervalues():
            if placement: placement.clearlayout()
        self.placementcache={}


    def goto(self, imageryprovider, loc, dist, screensize):

        if not imageryprovider: imageryprovider=None
        if imageryprovider!=self.imageryprovider:
            self.provider_base=None
            self.provider_url=None
            self.provider_logo=None
            self.imageryprovider=imageryprovider
            if self.imageryprovider not in self.providers: return
            self.q.put((self.providers[self.imageryprovider], ()))

        newtile=(int(floor(loc[0])),int(floor(loc[1])))
        if not self.provider_url or self.tile!=newtile:
            # New tile - drop cache of Clutter
            for placement in self.placementcache.itervalues():
                if placement: placement.clearlayout()
            self.placementcache={}
        self.tile=newtile
        self.loc=loc
        self.placements(dist, screensize)	# Kick off any image loading


    # Return placements to be drawn. May allocate into vertexcache as a side effect.
#.........这里部分代码省略.........

# 需要导入模块: from Queue import LifoQueue [as 别名]
# 或者: from Queue.LifoQueue import task_done [as 别名]

#.........这里部分代码省略.........
                else:
                    if (events > 0):
                        msg = incoming.recv_multipart()
                        self.message_in(tuple(msg))

                    # Now recover and queued outgoing messages:
                    if not self._out_queue.empty():
                        message = self._out_queue.get_nowait()
                        if message:
                            try:
                                # send sync hub followed by message. The sync
                                # will kick the hub into life if its just
                                # started:
                                outgoing.send_multipart(self.sync_message)
                                outgoing.send_multipart(message)

                            except ZMQError as e:
                                # 4 = 'Interrupted system call'
                                if e.errno == 4:
                                    self.log.info((
                                        "main: sigint or other signal interrupt"
                                        ", exit time <%s>"
                                    ) % e)
                                    break
                                else:
                                    self.log.info("main: <%s>" % e)
                                    break

                            except Exception:
                                self.log.exception("main: fatal error sending ")
                                break

                            finally:
                                self._out_queue.task_done()

        finally:
            self.wait_for_exit.set()
            _shutdown()


    def start(self):
        """Set up zmq communication and start receiving messages from the hub.
        """
        # coverage can't seem to get to this:
        def _main(notused): # pragma: no cover
            self.exit_time.clear()
            self.wait_for_exit.clear()
            self.main()
        thread.start_new(_main, (0,))


    def stop(self, wait=2):
        """Stop receiving messages from the hub and clean up.

        :param wait: The time in seconds to wait before giving up
        on a clean shutdown.

        """
        self.log.info("stop: shutting down messaging.")
        self.exit_time.set()
        self.wait_for_exit.wait(wait)
        self.log.info("stop: done.")


    def message_out(self, message):
        """This sends a message to the messagehub for dispatch to all connected

# 需要导入模块: from Queue import LifoQueue [as 别名]
# 或者: from Queue.LifoQueue import task_done [as 别名]
class B2BucketThreadedLocal(B2Bucket): 
    def __init__(self, *args):
        super(B2BucketThreaded, self).__init__( *args)
        
        num_threads=50
        self.queue = LifoQueue(num_threads*2)
        
        self.file_locks = defaultdict(Lock)
        
        self.running = True
        
        self.threads = []
        print "Thread ",
        for i in xrange(num_threads):
            t = threading.Thread(target=self._file_updater)
            t.start()
            self.threads.append(t)
            
            print ".",
            
        print 
        
        self.pre_queue_lock = Lock()
        self.pre_queue_running = True
        self.pre_queue = LifoQueue(num_threads*2)
        
        self.pre_file_dict = {}
        
        self.pre_thread = threading.Thread(target=self._prepare_update)
        self.pre_thread.start()
        
        
    
    def _prepare_update(self):
        while self.pre_queue_running:
            try:
                filename, local_filename, operation  = self.pre_queue.get(True,1)
                self.pre_file_dict[filename] = (time(), local_filename, operation)
                self.pre_queue.task_done()
            except Empty:
                for filename, (timestamp, local_filename, operation) in self.pre_file_dict.items():
                    if time()-timestamp > 15:
                        self.queue.put((filename, local_filename, operation))
                    
                        del self.pre_file_dict[filename]
        
        for filename, (timestamp, local_filename, operation) in self.pre_file_dict.items():
            self.queue.put((filename, local_filename, operation))
            del self.pre_file_dict[filename]
            
    def _file_updater(self):
        while self.running:
            try:
                filename, local_filename, operation  = self.queue.get(True,1)
            except Empty:
                continue
            
            
            with self.file_locks[filename]:
                if operation == "deletion":
                    super(B2BucketThreaded,self)._delete_file(filename)
                    self.queue.task_done()
                    
                elif operation == "upload":
                    super(B2BucketThreaded,self)._put_file(filename, local_filename)
                    self.queue.task_done()
                    
                elif operation == "download":
                    super(B2BucketThreaded,self)._get_file(filename, local_filename)
                    self.queue.task_done()
                    
                else:
                    self.logger.error("Invalid operation %s on %s" % (operation, filename))
                
            
    
    def __enter__(self):
        return self
        
    def __exit__(self, *args, **kwargs):
        self.logger.info("Waiting for all B2 requests to complete")
        
        self.logger.info("Pre-Queue contains %s elements", self.pre_queue.qsize())
        self.pre_queue.join()
        
        self.logger.info("Joining pre queue thread")
        self.pre_queue_running = False
        self.pre_thread.join()
        
        self.logger.info("Queue contains %s elements", self.queue.qsize())
        self.queue.join()
        
        self.logger.info("Joining threads")
        self.running = False
        for t in self.threads:
            t.join()
            
            
    def put_file(self, filename, local_filename):
        with self.pre_queue_lock:
#.........这里部分代码省略.........