Python queue.JoinableQueue類代碼示例

class Speaker(gevent.Greenlet):
    RATE = 44100

    def __init__(self, rcv):
        gevent.Greenlet.__init__(self)
        self.rcv = rcv
        PA = pyaudio.PyAudio()
        self.pa= PA.open(
            format= pyaudio.paInt16,
            channels= 1,
            rate= self.RATE,
            output= True
        )
        self.queue = JoinableQueue()

    def _run(self):
        print 'spk_on'
        while True:
            try:
                buf = self.rcv.queue.get()
            except gevent.queue.Empty:
                buf = '\0'
##            print '.',
            self.pa.write(buf)
            time.sleep(0.0001)

        self.queue.task_done()
        self.pa.close()

    def __init__(self, *args, **kwargs):
        super(InterceptedStreamsMixin, self).__init__(*args, **kwargs)
        self.stdout_callbacks = []
        self.stdout_queue = JoinableQueue()

        self.stderr_callbacks = []
        self.stderr_queue = JoinableQueue()

def test_service():
    "Service() should implement the basic needs of an async service"

    # Given the following service
    class MyService(Service):
        def __init__(self, my_mock, result_queue=None):
            self.my_mock = my_mock
            super(MyService, self).__init__(
                callback=self.run,
                result_queue=result_queue,
            )

        def run(self, package, sender_data):
            self.my_mock.ran = package

    my_mock = Mock()
    queue = JoinableQueue()
    service = MyService(my_mock, result_queue=queue)

    # When I queue a package to be processed by my service and start the
    # service with 1 concurrent worker
    service.queue('gherkin==0.1.0', 'main')
    service.consume()

    # Then I see that the package processed
    package = queue.get()
    package.should.equal('gherkin==0.1.0')

    my_mock.ran.should.equal('gherkin==0.1.0')

def test_service_failure():
    "Service() should handle custom callback failures"

    # Given the following service
    class MyService(Service):
        def __init__(self, result_queue=None):
            super(MyService, self).__init__(
                callback=self.run,
                result_queue=result_queue,
            )

        def run(self, package, sender_data):
            raise ValueError("I don't want to do anything")

    queue = JoinableQueue()
    service = MyService(result_queue=queue)

    # When I queue a package to be processed by my service and start the
    # service with 1 concurrent worker
    service.queue('gherkin==0.1.0', 'main')
    service.consume()
    service.pool.join()         # Ensure we finish spawning the greenlet

    # Then I see that no package was processed
    queue.qsize().should.equal(0)

    # And that the list of failed packages was updated
    service.failed_queue[0][0].should.equal('gherkin==0.1.0')
    service.failed_queue[0][1].should.be.a(ValueError)
    service.failed_queue[0][1].message.should.equal("I don't want to do anything")

def handle():
    connection = create_postgresql_connection()

    cursor = connection.cursor()
    cursor.execute("BEGIN;")
    cursor.execute("DELETE FROM core_ratequery;")
    cursor.execute("COMMIT;")
    cursor.close()

    queue = JoinableQueue()
    event = Event()

    age_ids = age_map(connection).values() + [None]
    sex_ids = sex_map(connection).values() + [None]
    education_ids = education_map(connection).values() + [None]
    province_ids = province_map(connection).values() + [None]

    cursor = connection.cursor()
    cursor.execute("SELECT DISTINCT cycle FROM core_microdata;");
    cycles = [row[0] for row in cursor]
    cursor.close()

    greenlets = []

    for i in range(50):
        gv = gevent.spawn(worker, queue, event)
        greenlets.append(gv)

    combs = itertools.product(age_ids, sex_ids, province_ids, education_ids, cycles)
    for c in combs:
        queue.put(c)

    queue.join()
    event.set()
    gevent.joinall(greenlets)

 def __init__(self, *args, **kwargs):
     super(WebServer, self).__init__(*args, **kwargs)
     print 'Webserver started'
     self.debug = True
     self.cmd_queue = JoinableQueue()
     self.event_queue = JoinableQueue()
     self.cmd_id = 0
     self.cmd_results = {}
     gevent.spawn(self.send_commands_to_debugger)
     gevent.spawn(self.receive_events_from_debugger)

    def start(self):
        if not self.__threads:
            self.__threads = len(IPNetwork(self.__ip)) if len(IPNetwork(self.__ip)) <= 10 else 10
        if len(IPNetwork(self.__ip)) < int(self.__threads):
            print "Please decrease number of threads to number of hosts <= %s" % len(IPNetwork(self.__ip))
            exit()

        queue = JoinableQueue()
        [queue.put(str(ip)) for ip in IPNetwork(self.__ip)]

        workers = [spawn(self.get_ip_info, queue, self.__apis) for t in range(int(self.__threads))]

        queue.join()

    def test_main(self):
        queue = JoinableQueue()
        print dir(queue)
        queue.put(1)
        queue.put(3)
        queue.put(2)
        queue.put(6)
        print queue.qsize()

        print '1', queue.get(), queue.get()

 def __init__(self, requestHandler=BaseRequestHandler(),
                    parseHandler=BaseParseHandler(),
                    sheduler=BaseScheduler(),
                    pipeline=BasePipeline()):
     self.requestHandler = requestHandler
     self.parseHandler = parseHandler
     self.sheduler = sheduler
     self.pipeline = pipeline
     self.task_queue = JoinableQueue()
     self.response_queue = JoinableQueue()
     self.tasks_cnt = 0
     self.result_queue = JoinableQueue()
     self.jobs_cnt = config.num_threads
     self.start_time = time.time()
     self.stop = False

    def __init__(self,url,parameter,HTTPClients,ClientConnectionPool,task=None):

        if task is not None:
            self.celeryTask = task
            self.celeryTaskId = task.request.id
        else:
            self.celeryTask = None

        self.parameter = parameter
        self.url = url
        self.numberHTTPClients = HTTPClients
        self.numberClientConnectionPool = ClientConnectionPool

        self.http = HTTPClient.from_url(URL(url),concurrency=self.numberClientConnectionPool)
        self.clientPool = gevent.pool.Pool(self.numberHTTPClients)
        self.workQueue = JoinableQueue()

        self.resultList = {}
        self.workQueueMax = 0
        self.workQueueDone = 0
        self.countRequests = 0
        self.status_codes = {}
        self.status_codes_count = {}
        self.meta = {}

        self.greenletList = {}
        self.initAdditionalStructures()
        self.progressMeta = None

        self.exitFlag = False
        self.pauseRequests = False

    def __init__(self, scheme, create_devices=True,
                 write_data=True,
                 start_date="2000-01-01T00:00:00Z",
                 end_date="2014-12-31T00:00:00Z",
                 pool_size=3):
        self.scheme = scheme
        self.create_devices = create_devices
        self.should_write_data = write_data
        self.start_date = start_date
        self.end_date = end_date
        self.tdb = TDBClient(scheme.db_key, scheme.db_key,
                             scheme.db_secret,
                             base_url=scheme.db_baseurl)

        iq_endpoint = HTTPEndpoint(scheme.iq_baseurl,
                                   scheme.iq_key,
                                   scheme.iq_secret)
        self.tiq = TIQClient(iq_endpoint)
        self.queue = JoinableQueue()
        self.lock = Lock()
        self.dp_count = 0
        self.req_count = 0
        self.dp_reset = time.time()
        for i in range(pool_size):
            gevent.spawn(self.worker)

 def __init__(self, creds, pool_size=POOL_SIZE):
     self.client = get_session(creds['host'],
                               creds['key'],
                               creds['secret'])
     self.queue = JoinableQueue(maxsize=POOL_SIZE*2)
     for i in range(pool_size):
         gevent.spawn(self.worker)

 def _setup_command_system(self):
     # we have two refs to the commands queue,
     # but write_commands_q will switch to throwaway
     # after we receive a finish command
     self._read_commands_q = JoinableQueue(None)
     self._write_commands_q = self._read_commands_q 
     for x in range(self._workers_to_start):
         gevent.spawn(self._process_commands)

class GQueue(object):
    def __init__(self):
        self.__QUEUE = JoinableQueue()

    def job(self, func):
        @functools.wraps(func)
        def f(*args, **kwargs):
            self.__QUEUE.put([func, args, kwargs])

        return f

    def join(self):
        self.__QUEUE.join()

    def work(self):
        while True:
            func, args, kwargs = self.__QUEUE.get()
            try:
                func(*args, **kwargs)
            finally:
                self.__QUEUE.task_done()

    def run_worker(self, num=1):
        for i in range(num):
            gevent.spawn(self.work)

    def _run(self):
        paths = glob.glob(self.path)
        while not paths:
            gevent.sleep(0.01)
            paths = glob.glob(self.path)
        q = JoinableQueue()

        self.logger.debug('Tailing %s' % ', '.join(paths))
        self.tails = [Tail(p, q, self.statedir) for p in paths]

        while True:
            data = q.get()
            if data:
                if data.endswith('\n'):
                    data = data[0:-1]
                self.logger.debug('Received: %r' % data)
                self.output.put(Event(data=data))
            q.task_done()