Python Queue.task_done方法代码示例

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import task_done [as 别名]
def listener(file : str, workQueue : Queue):
    try:
        f = open(file, 'w', encoding='utf-8')
        f.write("URL|IS_HTTPS|HTTP_HEADER|RESULT\n")
        while 1:
            m = workQueue.get()
            workQueue.task_done()
            if m == 'kill':
                break
            f.write(str(m) + '\n')
            f.flush()
    except BaseException:
        print(sys.exc_info())
        raise
    finally:
        f.close()

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import task_done [as 别名]
class _WorkerClass(object):
    _use_processes = True
    
    def __init__(self, *args, **kwargs):
        if 'use_processes' in kwargs:
            self._use_processes = kwargs['use_processes']
            del kwargs['use_processes']
        
    def _setup_main_queue(self):
        self._queue = Queue()
        
        if self._use_processes is True:
            t = Process(target=self._process_queue)
        else:
            t = _Thread(target=self._process_queue)
        t.name = "Queue"
        t.daemon = True
        t.start()
        print "Queue started", t
    
    def _process_queue(self):
        while 1:
            print "Processing queue... Size is", self._queue.qsize()
            request = self._queue.get()
            print "Request is:",request
            self._do_request(request)
            self._queue.task_done()
    
    def _do_request(self, function, *args, **kwargs):
        if type(function) == type(()):
            function, args, kwargs = function[:]
        if 'callback' in kwargs:
            callback = kwargs['callback']
            del kwargs['callback']
        else:
            callback = None

        if 'error' in kwargs:
            error = kwargs['error']
            del kwargs['error']
        else:
            error = None
        
        print "Run", function, args, kwargs
        try:
            r = function(*args, **kwargs)
            if not isinstance(r, tuple): r = (r,)
            if callback: self._do_callback(callback, *r)
        except Exception as e:
            print "Error occured"
            if error:
                self._do_callback(error, e)
            else:
                print "There was an error running the threaded function:", e
                print _print_exception_details()
    
    def _start_request_in_new_thread(self, function, *args, **kwargs):
        if self._use_processes is True:
            t = Process(target=self.__do_request, args=(function,)+args, kwargs=kwargs)
        else:
            t = _Thread(target=self.__do_request, args=(function,)+args, kwargs=kwargs)
        t.daemon = True
        t.start()
    
    def _add_request_to_main_queue(self, function, *args, **kwargs):
        try:
            getattr(self, '_queue')
        except AttributeError:
            self._setup_main_queue()

        self._queue.put((function, args, kwargs))
        print "Request added to queue. Size is now", self._queue.qsize()
    
    def _do_callback(self, callback, *args):
        def _callback(callback, args):
            callback(*args)
            return False

        _callback(callback, args)

# 需要导入模块: from multiprocessing import Queue [as 别名]
# 或者: from multiprocessing.Queue import task_done [as 别名]
class Mailbox:
    """
    Provides messaging functionality for a paxos system instance.
    """

    def __init__(self, config):
        self.config = config
        self.funnel = Queue()
        self.inbox = [Queue() for i in range(config.num_processes)]
        self.message_count = 0

        # Two flags, active to signal when we haven't received any messages
        # for timeout_interval seconds, and terminate to signal when we have
        # been asked to shutdown by the System.
        self.active = True
        self.terminate = False
        # If don't receive any messages in this amount of time, then shutdown.
        self.timeout_interval = 3 * self.config.message_timeout
        # Time stamp of the last seen message, used together with
        # timeout_interval to determine when the mailbox should shutdown.
        self.last_seen = None

    def run(self):
        print("Mailbox started")
        while True:
            if not self.active and self.terminate:
                break
            if self.active and self.last_seen and (time.time() - self.last_seen) > self.timeout_interval:
                self.active = False
            # Take messages off the funnel and deliver them to the appropriate
            # process.
            try:
                dest, msg = self.funnel.get(timeout=0.5)
                self.last_seen = time.time()
            except queue.Empty:
                pass
            else:
                self.inbox[dest].put(msg)
        print("Mailbox shutting down")

    def send(self, to, msg):
        """
        Send msg to process id ``to``.
        """
        # Funnel all messages through a primary queue so that we can keep track
        # of when we are done (i.e. all messages are processed).
        self.message_count += 1
        self.funnel.put((to, msg))

    def recv(self, from_):
        """
        Receive (blocking) msg destined for process id ``from_``.
        """
        return self.inbox[from_].get()

    def task_done(self, pid):
        """
        Inform pid's queue that it has processed a task.
        Called by agents when they are done processing a message, and used by
        the queue instance to know when all messages have been processed.
        """
        self.funnel.task_done()

    def join(self):
        """
        Block until all messages have finished processing and we haven't had
        any messages for a while (i.e. active set to False).
        """
        while self.active:
            time.sleep(0.5)
        # Don't join funnel queue because there's a good chance that it will
        # never be fully exhausted due to heart beat messages.
        #self.funnel.join()

    def shutdown(self):
        """
        Perform any shutdown actions prior to quitting.  In this base Mailbox
        class this is just a hook that isn't used.
        """
        pass

    def quit(self):
        self.terminate = True