Python Queue.get_nowait方法代码示例

# 需要导入模块: from polyglot.queue import Queue [as 别名]
# 或者: from polyglot.queue.Queue import get_nowait [as 别名]
class GenericDownloadThreadPool(object):
    '''
    add_task must be implemented in a subclass and must
    GenericDownloadThreadPool.add_task must be called
    at the end of the function.
    '''

    def __init__(self, thread_type, thread_count=1):
        self.thread_type = thread_type
        self.thread_count = thread_count

        self.tasks = Queue()
        self.results = Queue()
        self.threads = []

    def set_thread_count(self, thread_count):
        self.thread_count = thread_count

    def add_task(self):
        '''
        This must be implemented in a sub class and this function
        must be called at the end of the add_task function in
        the sub class.

        The implementation of this function (in this base class)
        starts any threads necessary to fill the pool if it is
        not already full.
        '''
        for i in range(self.thread_count - self.running_threads_count()):
            t = self.thread_type(self.tasks, self.results)
            self.threads.append(t)
            t.start()

    def abort(self):
        self.tasks = Queue()
        self.results = Queue()
        for t in self.threads:
            t.abort()
        self.threads = []

    def has_tasks(self):
        return not self.tasks.empty()

    def get_result(self):
        return self.results.get()

    def get_result_no_wait(self):
        return self.results.get_nowait()

    def result_count(self):
        return len(self.results)

    def has_results(self):
        return not self.results.empty()

    def threads_running(self):
        return self.running_threads_count() > 0

    def running_threads_count(self):
        count = 0
        for t in self.threads:
            if t.is_alive():
                count += 1
        return count

# 需要导入模块: from polyglot.queue import Queue [as 别名]
# 或者: from polyglot.queue.Queue import get_nowait [as 别名]
def test_identify_plugin(name, tests, modify_plugin=lambda plugin:None,  # {{{
        fail_missing_meta=True):
    '''
    :param name: Plugin name
    :param tests: List of 2-tuples. Each two tuple is of the form (args,
                  test_funcs). args is a dict of keyword arguments to pass to
                  the identify method. test_funcs are callables that accept a
                  Metadata object and return True iff the object passes the
                  test.
    '''
    plugin = None
    for x in all_metadata_plugins():
        if x.name == name and 'identify' in x.capabilities:
            plugin = x
            break
    modify_plugin(plugin)
    prints('Testing the identify function of', plugin.name)
    prints('Using extra headers:', plugin.browser.addheaders)

    tdir, lf, log, abort = init_test(plugin.name)
    prints('Log saved to', lf)

    times = []
    for kwargs, test_funcs in tests:
        log('')
        log('#'*80)
        log('### Running test with:', kwargs)
        log('#'*80)
        prints('Running test with:', kwargs)
        rq = Queue()
        args = (log, rq, abort)
        start_time = time.time()
        plugin.running_a_test = True
        try:
            err = plugin.identify(*args, **kwargs)
        finally:
            plugin.running_a_test = False
        total_time = time.time() - start_time
        times.append(total_time)
        if err is not None:
            prints('identify returned an error for args', args)
            prints(err)
            break

        results = []
        while True:
            try:
                results.append(rq.get_nowait())
            except Empty:
                break

        prints('Found', len(results), 'matches:', end=' ')
        prints('Smaller relevance means better match')

        results.sort(key=plugin.identify_results_keygen(
            title=kwargs.get('title', None), authors=kwargs.get('authors',
                None), identifiers=kwargs.get('identifiers', {})))

        for i, mi in enumerate(results):
            prints('*'*30, 'Relevance:', i, '*'*30)
            if mi.rating:
                mi.rating *= 2
            prints(mi)
            prints('\nCached cover URL    :',
                    plugin.get_cached_cover_url(mi.identifiers))
            prints('*'*75, '\n\n')

        possibles = []
        for mi in results:
            test_failed = False
            for tfunc in test_funcs:
                if not tfunc(mi):
                    test_failed = True
                    break
            if not test_failed:
                possibles.append(mi)

        if not possibles:
            prints('ERROR: No results that passed all tests were found')
            prints('Log saved to', lf)
            log.close()
            dump_log(lf)
            raise SystemExit(1)

        good = [x for x in possibles if plugin.test_fields(x) is
                None]
        if not good:
            prints('Failed to find', plugin.test_fields(possibles[0]))
            if fail_missing_meta:
                raise SystemExit(1)

        if results[0] is not possibles[0]:
            prints('Most relevant result failed the tests')
            raise SystemExit(1)

        if 'cover' in plugin.capabilities:
            rq = Queue()
            mi = results[0]
            plugin.download_cover(log, rq, abort, title=mi.title,
                    authors=mi.authors, identifiers=mi.identifiers)
#.........这里部分代码省略.........

# 需要导入模块: from polyglot.queue import Queue [as 别名]
# 或者: from polyglot.queue.Queue import get_nowait [as 别名]
class ParseWorker(Thread):

    daemon = True
    SLEEP_TIME = 1

    def __init__(self):
        Thread.__init__(self)
        self.requests = Queue()
        self.request_count = 0
        self.parse_items = {}
        self.launch_error = None

    def run(self):
        mod, func = 'calibre.gui2.tweak_book.preview', 'parse_html'
        try:
            # Connect to the worker and send a dummy job to initialize it
            self.worker = offload_worker(priority='low')
            self.worker(mod, func, '<p></p>')
        except:
            import traceback
            traceback.print_exc()
            self.launch_error = traceback.format_exc()
            return

        while True:
            time.sleep(self.SLEEP_TIME)
            x = self.requests.get()
            requests = [x]
            while True:
                try:
                    requests.append(self.requests.get_nowait())
                except Empty:
                    break
            if shutdown in requests:
                self.worker.shutdown()
                break
            request = sorted(requests, reverse=True)[0]
            del requests
            pi, data = request[1:]
            try:
                res = self.worker(mod, func, data)
            except:
                import traceback
                traceback.print_exc()
            else:
                pi.parsing_done = True
                parsed_data = res['result']
                if res['tb']:
                    prints("Parser error:")
                    prints(res['tb'])
                else:
                    pi.parsed_data = parsed_data

    def add_request(self, name):
        data = get_data(name)
        ldata, hdata = len(data), hash(data)
        pi = self.parse_items.get(name, None)
        if pi is None:
            self.parse_items[name] = pi = ParseItem(name)
        else:
            if pi.parsing_done and pi.length == ldata and pi.fingerprint == hdata:
                return
            pi.parsed_data = None
            pi.parsing_done = False
        pi.length, pi.fingerprint = ldata, hdata
        self.requests.put((self.request_count, pi, data))
        self.request_count += 1

    def shutdown(self):
        self.requests.put(shutdown)

    def get_data(self, name):
        return getattr(self.parse_items.get(name, None), 'parsed_data', None)

    def clear(self):
        self.parse_items.clear()

    def is_alive(self):
        return Thread.is_alive(self) or (hasattr(self, 'worker') and self.worker.is_alive())

# 需要导入模块: from polyglot.queue import Queue [as 别名]
# 或者: from polyglot.queue.Queue import get_nowait [as 别名]
class WebSocketConnection(HTTPConnection):

    # Internal API {{{
    in_websocket_mode = False
    websocket_handler = None

    def __init__(self, *args, **kwargs):
        global conn_id
        HTTPConnection.__init__(self, *args, **kwargs)
        self.sendq = Queue()
        self.control_frames = deque()
        self.cf_lock = Lock()
        self.sending = None
        self.send_buf = None
        self.frag_decoder = UTF8Decoder()
        self.ws_close_received = self.ws_close_sent = False
        conn_id += 1
        self.websocket_connection_id = conn_id
        self.stop_reading = False

    def finalize_headers(self, inheaders):
        upgrade = inheaders.get('Upgrade', '')
        key = inheaders.get('Sec-WebSocket-Key', None)
        conn = {x.strip().lower() for x in inheaders.get('Connection', '').split(',')}
        if key is None or upgrade.lower() != 'websocket' or 'upgrade' not in conn:
            return HTTPConnection.finalize_headers(self, inheaders)
        ver = inheaders.get('Sec-WebSocket-Version', 'Unknown')
        try:
            ver_ok = int(ver) >= 13
        except Exception:
            ver_ok = False
        if not ver_ok:
            return self.simple_response(http_client.BAD_REQUEST, 'Unsupported WebSocket protocol version: %s' % ver)
        if self.method != 'GET':
            return self.simple_response(http_client.BAD_REQUEST, 'Invalid WebSocket method: %s' % self.method)

        response = HANDSHAKE_STR % as_base64_unicode(sha1((key + GUID_STR).encode('utf-8')).digest())
        self.optimize_for_sending_packet()
        self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
        self.set_state(WRITE, self.upgrade_connection_to_ws, ReadOnlyFileBuffer(response.encode('ascii')), inheaders)

    def upgrade_connection_to_ws(self, buf, inheaders, event):
        if self.write(buf):
            if self.websocket_handler is None:
                self.websocket_handler = DummyHandler()
            self.read_frame, self.current_recv_opcode = ReadFrame(), None
            self.in_websocket_mode = True
            try:
                self.websocket_handler.handle_websocket_upgrade(self.websocket_connection_id, weakref.ref(self), inheaders)
            except Exception as err:
                self.log.exception('Error in WebSockets upgrade handler:')
                self.websocket_close(UNEXPECTED_ERROR, 'Unexpected error in handler: %r' % as_unicode(err))
            self.handle_event = self.ws_duplex
            self.set_ws_state()
            self.end_send_optimization()

    def set_ws_state(self):
        if self.ws_close_sent or self.ws_close_received:
            if self.ws_close_sent:
                self.ready = False
            else:
                self.wait_for = WRITE
            return

        if self.send_buf is not None or self.sending is not None:
            self.wait_for = RDWR
        else:
            try:
                self.sending = self.sendq.get_nowait()
            except Empty:
                with self.cf_lock:
                    if self.control_frames:
                        self.wait_for = RDWR
                    else:
                        self.wait_for = READ
            else:
                self.wait_for = RDWR

        if self.stop_reading:
            if self.wait_for is READ:
                self.ready = False
            elif self.wait_for is RDWR:
                self.wait_for = WRITE

    def ws_duplex(self, event):
        if event is READ:
            self.ws_read()
        elif event is WRITE:
            self.ws_write()
        self.set_ws_state()

    def ws_read(self):
        if not self.stop_reading:
            self.read_frame(self)

    def ws_data_received(self, data, opcode, frame_starting, frame_finished, is_final_frame_of_message):
        if opcode in CONTROL_CODES:
            return self.ws_control_frame(opcode, data)

        message_starting = self.current_recv_opcode is None
#.........这里部分代码省略.........

# 需要导入模块: from polyglot.queue import Queue [as 别名]
# 或者: from polyglot.queue.Queue import get_nowait [as 别名]
class Server(Thread):

    def __init__(self, notify_on_job_done=lambda x: x, pool_size=None,
            limit=sys.maxsize, enforce_cpu_limit=True):
        Thread.__init__(self)
        self.daemon = True
        global _counter
        self.id = _counter+1
        _counter += 1

        if enforce_cpu_limit:
            limit = min(limit, cpu_count())
        self.pool_size = limit if pool_size is None else pool_size
        self.notify_on_job_done = notify_on_job_done
        self.auth_key = os.urandom(32)
        self.address, self.listener = create_listener(self.auth_key, backlog=4)
        self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue()
        self.kill_queue = Queue()
        self.waiting_jobs = []
        self.workers = deque()
        self.launched_worker_count = 0
        self._worker_launch_lock = RLock()

        self.start()

    def launch_worker(self, gui=False, redirect_output=None, job_name=None):
        start = time.time()
        with self._worker_launch_lock:
            self.launched_worker_count += 1
            id = self.launched_worker_count
        fd, rfile = tempfile.mkstemp(prefix=u'ipc_result_%d_%d_'%(self.id, id),
                dir=base_dir(), suffix=u'.pickle')
        os.close(fd)
        if redirect_output is None:
            redirect_output = not gui

        env = {
                'CALIBRE_WORKER_ADDRESS' : environ_item(as_hex_unicode(msgpack_dumps(self.address))),
                'CALIBRE_WORKER_KEY' : environ_item(as_hex_unicode(self.auth_key)),
                'CALIBRE_WORKER_RESULT' : environ_item(as_hex_unicode(rfile)),
              }
        cw = self.do_launch(env, gui, redirect_output, rfile, job_name=job_name)
        if isinstance(cw, string_or_bytes):
            raise CriticalError('Failed to launch worker process:\n'+cw)
        if DEBUG:
            print('Worker Launch took:', time.time() - start)
        return cw

    def do_launch(self, env, gui, redirect_output, rfile, job_name=None):
        w = Worker(env, gui=gui, job_name=job_name)

        try:
            w(redirect_output=redirect_output)
            conn = eintr_retry_call(self.listener.accept)
            if conn is None:
                raise Exception('Failed to launch worker process')
        except BaseException:
            try:
                w.kill()
            except:
                pass
            import traceback
            return traceback.format_exc()
        return ConnectedWorker(w, conn, rfile)

    def add_job(self, job):
        job.done2 = self.notify_on_job_done
        self.add_jobs_queue.put(job)

    def run_job(self, job, gui=True, redirect_output=False):
        w = self.launch_worker(gui=gui, redirect_output=redirect_output, job_name=getattr(job, 'name', None))
        w.start_job(job)

    def run(self):
        while True:
            try:
                job = self.add_jobs_queue.get(True, 0.2)
                if job is None:
                    break
                self.waiting_jobs.insert(0, job)
            except Empty:
                pass

            # Get notifications from worker process
            for worker in self.workers:
                while True:
                    try:
                        n = worker.notifications.get_nowait()
                        worker.job.notifications.put(n)
                        self.changed_jobs_queue.put(worker.job)
                    except Empty:
                        break

            # Remove finished jobs
            for worker in [w for w in self.workers if not w.is_alive]:
                try:
                    worker.close_log_file()
                except:
                    pass
                self.workers.remove(worker)
#.........这里部分代码省略.........

# 需要导入模块: from polyglot.queue import Queue [as 别名]
# 或者: from polyglot.queue.Queue import get_nowait [as 别名]

#.........这里部分代码省略.........
        if not gprefs.get('server3_warning_done', False):
            gprefs.set('server3_warning_done', True)
            if os.path.exists(os.path.join(config_dir, 'server.py')):
                try:
                    os.remove(os.path.join(config_dir, 'server.py'))
                except EnvironmentError:
                    pass
                warning_dialog(self, _('Content server changed!'), _(
                    'calibre 3 comes with a completely re-written content server.'
                    ' As such any custom configuration you have for the content'
                    ' server no longer applies. You should check and refresh your'
                    ' settings in Preferences->Sharing->Sharing over the net'), show=True)
        self.content_server = Server(self.library_broker, Dispatcher(self.handle_changes_from_server))
        self.content_server.state_callback = Dispatcher(
                self.iactions['Connect Share'].content_server_state_changed)
        if check_started:
            self.content_server.start_failure_callback = \
                Dispatcher(self.content_server_start_failed)
        self.content_server.start()

    def handle_changes_from_server(self, library_path, change_event):
        if DEBUG:
            prints('Received server change event: {} for {}'.format(change_event, library_path))
        if self.library_broker.is_gui_library(library_path):
            self.server_changes.put((library_path, change_event))
            self.server_change_notification_timer.start()

    def handle_changes_from_server_debounced(self):
        if self.shutting_down:
            return
        changes = []
        while True:
            try:
                library_path, change_event = self.server_changes.get_nowait()
            except Empty:
                break
            if self.library_broker.is_gui_library(library_path):
                changes.append(change_event)
        if changes:
            handle_changes(changes, self)

    def content_server_start_failed(self, msg):
        self.content_server = None
        error_dialog(self, _('Failed to start Content server'),
                _('Could not start the Content server. Error:\n\n%s')%msg,
                show=True)

    def resizeEvent(self, ev):
        MainWindow.resizeEvent(self, ev)
        self.search.setMaximumWidth(self.width()-150)

    def create_spare_pool(self, *args):
        if self._spare_pool is None:
            num = min(detect_ncpus(), int(config['worker_limit']/2.0))
            self._spare_pool = Pool(max_workers=num, name='GUIPool')

    def spare_pool(self):
        ans, self._spare_pool = self._spare_pool, None
        QTimer.singleShot(1000, self.create_spare_pool)
        return ans

    def do_proceed(self, func, payload):
        if callable(func):
            func(payload)

    def no_op(self, *args):

# 需要导入模块: from polyglot.queue import Queue [as 别名]
# 或者: from polyglot.queue.Queue import get_nowait [as 别名]

#.........这里部分代码省略.........
                    return (strftime(u'%H:%M -- %d %b', time.localtime(job.start_time)))
            if role == Qt.DecorationRole and col == 0:
                state = job.run_state
                if state == job.WAITING:
                    return self.wait_icon
                if state == job.RUNNING:
                    return self.running_icon
                if job.killed or job.failed:
                    return self.error_icon
                return self.done_icon
        except:
            import traceback
            traceback.print_exc()
        return None

    def update(self):
        try:
            self._update()
        except BaseException:
            import traceback
            traceback.print_exc()

    def _update(self):
        # Update running time
        for i, j in enumerate(self.jobs):
            if j.run_state == j.RUNNING:
                idx = self.index(i, 3)
                self.dataChanged.emit(idx, idx)

        # Update parallel jobs
        jobs = set([])
        while True:
            try:
                jobs.add(self.server.changed_jobs_queue.get_nowait())
            except Empty:
                break

        # Update device jobs
        while True:
            try:
                jobs.add(self.changed_queue.get_nowait())
            except Empty:
                break

        # Update threaded jobs
        while True:
            try:
                jobs.add(self.threaded_server.changed_jobs.get_nowait())
            except Empty:
                break

        if jobs:
            needs_reset = False
            for job in jobs:
                orig_state = job.run_state
                job.update()
                if orig_state != job.run_state:
                    needs_reset = True
                    if job.is_finished:
                        self.job_done.emit(len(self.unfinished_jobs()))
            if needs_reset:
                self.modelAboutToBeReset.emit()
                self.jobs.sort()
                self.modelReset.emit()
            else:
                for job in jobs:

# 需要导入模块: from polyglot.queue import Queue [as 别名]
# 或者: from polyglot.queue.Queue import get_nowait [as 别名]
class BaseJob(object):

    WAITING  = 0
    RUNNING  = 1
    FINISHED = 2

    def __init__(self, description, done=lambda x: x):
        self.id            = next(job_counter)
        self.description   = description
        self.done          = done
        self.done2         = None
        self.killed        = False
        self.failed        = False
        self.kill_on_start = False
        self.start_time    = None
        self.result        = None
        self.duration      = None
        self.log_path      = None
        self.notifications = Queue()

        self._run_state    = self.WAITING
        self.percent       = 0
        self._message      = None
        self._status_text  = _('Waiting...')
        self._done_called  = False
        self.core_usage    = 1
        self.timed_out     = False

    def update(self, consume_notifications=True):
        if self.duration is not None:
            self._run_state   = self.FINISHED
            self.percent = 100
            if self.killed:
                if self.timed_out:
                    self._status_text = _('Aborted, taking too long')
                else:
                    self._status_text = _('Stopped')
            else:
                self._status_text = _('Error') if self.failed else _('Finished')
            if DEBUG:
                try:
                    prints('Job:', self.id, self.description, 'finished',
                        safe_encode=True)
                    prints('\t'.join(self.details.splitlines(True)),
                        safe_encode=True)
                except:
                    pass
            if not self._done_called:
                self._done_called = True
                try:
                    self.done(self)
                except:
                    pass
                try:
                    if callable(self.done2):
                        self.done2(self)
                except:
                    pass
        elif self.start_time is not None:
            self._run_state = self.RUNNING
            self._status_text = _('Working...')

        if consume_notifications:
            return self.consume_notifications()
        return False

    def consume_notifications(self):
        got_notification = False
        while self.notifications is not None:
            try:
                self.percent, self._message = self.notifications.get_nowait()
                self.percent *= 100.
                got_notification = True
            except Empty:
                break
        return got_notification

    @property
    def status_text(self):
        if self._run_state == self.FINISHED or not self._message:
            return self._status_text
        return self._message

    @property
    def run_state(self):
        return self._run_state

    @property
    def running_time(self):
        if self.duration is not None:
            return self.duration
        if self.start_time is not None:
            return time.time() - self.start_time
        return None

    @property
    def is_finished(self):
        return self._run_state == self.FINISHED

    @property
#.........这里部分代码省略.........