Python queue.Queue类代码示例

def single_covers(title, authors, identifiers, caches, tdir):
    patch_plugins()
    load_caches(caches)
    log = GUILog()
    results = Queue()
    worker = Thread(target=run_download, args=(log, results, Event()),
            kwargs=dict(title=title, authors=authors, identifiers=identifiers))
    worker.daemon = True
    worker.start()
    c = Counter()
    while worker.is_alive():
        try:
            plugin, width, height, fmt, data = results.get(True, 1)
        except Empty:
            continue
        else:
            name = plugin.name
            if plugin.can_get_multiple_covers:
                name += '{%d}'%c[plugin.name]
                c[plugin.name] += 1
            name = '%s,,%s,,%s,,%s.cover'%(name, width, height, fmt)
            with open(os.path.join(tdir, name), 'wb') as f:
                f.write(data)
            os.mkdir(os.path.join(tdir, name+'.done'))

    return log.dump()

    def __init__(self, thread_type, thread_count=1):
        self.thread_type = thread_type
        self.thread_count = thread_count

        self.tasks = Queue()
        self.results = Queue()
        self.threads = []

    class Watcher(WatcherBase):

        def __init__(self, root_dirs, worker, log):
            WatcherBase.__init__(self, worker, log)
            self.stream = Stream(self.notify, *(x.encode('utf-8') for x in root_dirs), file_events=True)
            self.wait_queue = Queue()

        def wakeup(self):
            self.wait_queue.put(True)

        def loop(self):
            observer = Observer()
            observer.schedule(self.stream)
            observer.daemon = True
            observer.start()
            try:
                while True:
                    try:
                        # Cannot use blocking get() as it is not interrupted by
                        # Ctrl-C
                        if self.wait_queue.get(10000) is True:
                            self.force_restart()
                    except Empty:
                        pass
            finally:
                observer.unschedule(self.stream)
                observer.stop()

        def notify(self, ev):
            name = ev.name
            if isinstance(name, bytes):
                name = name.decode('utf-8')
            if self.file_is_watched(name):
                self.handle_modified({name})

class DBThread(Thread):

    CLOSE = '-------close---------'

    def __init__(self, path, row_factory):
        Thread.__init__(self)
        self.setDaemon(True)
        self.path = path
        self.unhandled_error = (None, '')
        self.row_factory = row_factory
        self.requests = Queue(1)
        self.results  = Queue(1)
        self.conn = None

    def connect(self):
        self.conn = do_connect(self.path, self.row_factory)

    def run(self):
        try:
            self.connect()
            while True:
                func, args, kwargs = self.requests.get()
                if func == self.CLOSE:
                    self.conn.close()
                    break
                if func == 'dump':
                    try:
                        ok, res = True, tuple(self.conn.iterdump())
                    except Exception as err:
                        ok, res = False, (err, traceback.format_exc())
                elif func == 'create_dynamic_filter':
                    try:
                        f = DynamicFilter(args[0])
                        self.conn.create_function(args[0], 1, f)
                        ok, res = True, f
                    except Exception as err:
                        ok, res = False, (err, traceback.format_exc())
                else:
                    bfunc = getattr(self.conn, func)
                    try:
                        for i in range(3):
                            try:
                                ok, res = True, bfunc(*args, **kwargs)
                                break
                            except OperationalError as err:
                                # Retry if unable to open db file
                                e = str(err)
                                if 'unable to open' not in e or i == 2:
                                    if 'unable to open' in e:
                                        prints('Unable to open database for func',
                                            func, reprlib.repr(args),
                                            reprlib.repr(kwargs))
                                    raise
                            time.sleep(0.5)
                    except Exception as err:
                        ok, res = False, (err, traceback.format_exc())
                self.results.put((ok, res))
        except Exception as err:
            self.unhandled_error = (err, traceback.format_exc())

 def __init__(self, path, row_factory):
     Thread.__init__(self)
     self.setDaemon(True)
     self.path = path
     self.unhandled_error = (None, '')
     self.row_factory = row_factory
     self.requests = Queue(1)
     self.results  = Queue(1)
     self.conn = None

class ConnectedWorker(Thread):

    def __init__(self, worker, conn, rfile):
        Thread.__init__(self)
        self.daemon = True
        self.conn = conn
        self.worker = worker
        self.notifications = Queue()
        self._returncode = 'dummy'
        self.killed = False
        self.log_path = worker.log_path
        self.rfile = rfile
        self.close_log_file = getattr(worker, 'close_log_file', None)

    def start_job(self, job):
        notification = PARALLEL_FUNCS[job.name][-1] is not None
        eintr_retry_call(self.conn.send, (job.name, job.args, job.kwargs, job.description))
        if notification:
            self.start()
        else:
            self.conn.close()
        self.job = job

    def run(self):
        while True:
            try:
                x = eintr_retry_call(self.conn.recv)
                self.notifications.put(x)
            except BaseException:
                break
        try:
            self.conn.close()
        except BaseException:
            pass

    def kill(self):
        self.killed = True
        try:
            self.worker.kill()
        except BaseException:
            pass

    @property
    def is_alive(self):
        return not self.killed and self.worker.is_alive

    @property
    def returncode(self):
        if self._returncode != 'dummy':
            return self._returncode
        r = self.worker.returncode
        if self.killed and r is None:
            self._returncode = 1
            return 1
        if r is not None:
            self._returncode = r
        return r

    def __init__(self, max_workers=None, name=None):
        Thread.__init__(self, name=name)
        self.max_workers = max_workers or detect_ncpus()
        self.available_workers = []
        self.busy_workers = {}
        self.pending_jobs = []
        self.events = Queue()
        self.results = Queue()
        self.tracker = Queue()
        self.terminal_failure = None
        self.common_data = pickle_dumps(None)
        self.worker_data = None
        self.shutting_down = False

        self.start()

    def __init__(self):
        Thread.__init__(self)
        self.daemon = True
        self.lock = RLock()

        self.queued_jobs = []
        self.running_jobs = set()
        self.changed_jobs = Queue()
        self.keep_going = True

 def __init__(self, ps1='>>> ', ps2='... ', show_js=False, libdir=None):
     Thread.__init__(self, name='RapydScriptREPL')
     self.to_python = to_python
     self.JSError = JSError
     self.enc = getattr(sys.stdin, 'encoding', None) or 'utf-8'
     try:
         import readline
         self.readline = readline
     except ImportError:
         pass
     self.output = ANSIStream(sys.stdout)
     self.to_repl = Queue()
     self.from_repl = Queue()
     self.ps1, self.ps2 = ps1, ps2
     self.show_js, self.libdir = show_js, libdir
     self.prompt = ''
     self.completions = None
     self.start()

def compress_images(container, report=None, names=None, jpeg_quality=None, progress_callback=lambda n, t, name:True):
    images = get_compressible_images(container)
    if names is not None:
        images &= set(names)
    results = {}
    queue = Queue()
    abort = Event()
    for name in images:
        queue.put(name)

    def pc(name):
        keep_going = progress_callback(len(results), len(images), name)
        if not keep_going:
            abort.set()
    progress_callback(0, len(images), '')
    [Worker(abort, 'CompressImage%d' % i, queue, results, container, jpeg_quality, pc) for i in range(min(detect_ncpus(), len(images)))]
    queue.join()
    before_total = after_total = 0
    changed = False
    for name, (ok, res) in iteritems(results):
        name = force_unicode(name, filesystem_encoding)
        if ok:
            before, after = res
            if before != after:
                changed = True
            before_total += before
            after_total += after
            if report:
                if before != after:
                    report(_('{0} compressed from {1} to {2} bytes [{3:.1%} reduction]').format(
                        name, human_readable(before), human_readable(after), (before - after)/before))
                else:
                    report(_('{0} could not be further compressed').format(name))
        else:
            report(_('Failed to process {0} with error:').format(name))
            report(res)
    if report:
        if changed:
            report('')
            report(_('Total image filesize reduced from {0} to {1} [{2:.1%} reduction]').format(
                human_readable(before_total), human_readable(after_total), (before_total - after_total)/before_total))
        else:
            report(_('Images are already fully optimized'))
    return changed, results

 def __init__(self, worker, conn, rfile):
     Thread.__init__(self)
     self.daemon = True
     self.conn = conn
     self.worker = worker
     self.notifications = Queue()
     self._returncode = 'dummy'
     self.killed = False
     self.log_path = worker.log_path
     self.rfile = rfile
     self.close_log_file = getattr(worker, 'close_log_file', None)

    def __init__(self, log, abort, title, authors, identifiers, caches):
        Thread.__init__(self)
        self.daemon = True

        self.log, self.abort = log, abort
        self.title, self.authors, self.identifiers = (title, authors,
                identifiers)
        self.caches = caches

        self.rq = Queue()
        self.error = None

class Progress(Thread):

    def __init__(self, conn):
        Thread.__init__(self)
        self.daemon = True
        self.conn = conn
        self.queue = Queue()

    def __call__(self, percent, msg=''):
        self.queue.put((percent, msg))

    def run(self):
        while True:
            x = self.queue.get()
            if x is None:
                break
            try:
                eintr_retry_call(self.conn.send, x)
            except:
                break

 def __init__(self, *args, **kwargs):
     global conn_id
     HTTPConnection.__init__(self, *args, **kwargs)
     self.sendq = Queue()
     self.control_frames = deque()
     self.cf_lock = Lock()
     self.sending = None
     self.send_buf = None
     self.frag_decoder = UTF8Decoder()
     self.ws_close_received = self.ws_close_sent = False
     conn_id += 1
     self.websocket_connection_id = conn_id
     self.stop_reading = False

 def __init__(self, result_callback=lambda x:x, worker_entry_point='main'):
     Thread.__init__(self)
     self.worker_entry_point = worker_entry_point
     self.start()
     self.main_queue = Queue()
     self.result_callback = result_callback
     self.reap_thread = None
     self.shutting_down = False
     self.connected = Event()
     self.current_completion_request = None
     self.latest_completion_request_id = None
     self.request_count = 0
     self.lock = RLock()