Python ThreadPool.join方法代码示例

# 需要导入模块: from threadpool import ThreadPool [as 别名]
# 或者: from threadpool.ThreadPool import join [as 别名]
    def run(self, env, json_writer):
        '''
        Schedule all tests in profile for execution.

        See ``Test.schedule`` and ``Test.run``.
        '''

        self.prepare_test_list(env)

        # If using concurrency, add all the concurrent tests to the pool and
        # execute that pool
        if env.concurrent:
            pool = ThreadPool(multiprocessing.cpu_count())
            for (path, test) in self.test_list.items():
                if test.runConcurrent:
                    pool.add(test.execute, (env, path, json_writer))
            pool.join()

        # Run any remaining tests serially from a single thread pool after the
        # concurrent tests have finished
        pool = ThreadPool(1)
        for (path, test) in self.test_list.items():
            if not env.concurrent or not test.runConcurrent:
                pool.add(test.execute, (env, path, json_writer))
        pool.join()

# 需要导入模块: from threadpool import ThreadPool [as 别名]
# 或者: from threadpool.ThreadPool import join [as 别名]
def processing(tables):
    import opts
    options, _ = opts.parse_options()

    master_connect_setting, slaver_connect_setting = opts.parse_connection(options)

    master = SqlPool(**master_connect_setting)
    slaver = SqlPool(**slaver_connect_setting) 
    
    if not tables and options.tables:
        tables = options.tables

    if not tables:
        tables = show_tables(slaver)

    pool = ThreadPool(options.thread)

    for tb in tables: 
        ignored = False
        for i in options.tables_ignored:
            if tb.startswith(i):
               ignored = True
               break

        if ignored: continue
        if tb not in ['qing_filerecycle', 'qing_file_storage'] and \
           options.checksum and checksum_table(master, slaver, tb) == True:
                logging.warn("Table[%s] on slave was the smae as master . OK", tb)
            continue

        checker = TableChecker(master, slaver, tb, 
                    setp = 1000,
                    used_select_all = bool(tb in options.tables_select_all),
                    fix_diff=options.fix_diff)

        pool.put(checker.run)

    pool.join()

# 需要导入模块: from threadpool import ThreadPool [as 别名]
# 或者: from threadpool.ThreadPool import join [as 别名]
def worker_routine(context, idx):
    """ Worker routine """
    global worker_exe_path
    # Socket to talk to dispatcher
    socket0 = context.socket(zmq.REP)

    socket0.connect(url_worker)
    worker_id = 'Worker-%s' % idx
    socket0.setsockopt(zmq.IDENTITY, worker_id)

    socket_exit = context.socket(zmq.SUB)
    socket_exit.connect(url_worker_cmd)
    socket_exit.setsockopt(zmq.SUBSCRIBE, '')
    poller = zmq.Poller()
    poller.register(socket0, zmq.POLLIN)
    poller.register(socket_exit, zmq.POLLIN)

    from threadpool import ThreadPool, WorkRequest
    thread_pool = ThreadPool(thread_num_of_worker)

    def thread_work(client_exe_path, msg):
        # create a process to execute the client_exe
        # TODO: add execute timeout
        _cmd = '%s %s' % (client_exe_path, msg)
        cmd = WrapCommand(_cmd)
        cmd.start()
        logger.debug('Start command start %s' % _cmd)
        cmd.join()
        logger.debug('Start command end %s' % _cmd)
        return cmd

    def thread_work_cb(request, ret):
        if ret.returncode == 0:
            # run cmd process in thread success
            logger.debug("execute %s success %s" % (request.args[0], ret.returncode))
        else:
            logger.error("execute %s failed %s \n%s\n" % (request.args[0], ret.returncode, ' '.join(ret.results)))

    def handle_exception(request, exc_info):
        if not isinstance(exc_info, tuple):
            # Something is seriously wrong...
            logger.debug(request)
            logger.debug(exc_info)
            raise SystemExit
        logger.debug("**** Exception occured in request #%s: %s" % \
                (request.requestID, exc_info))

    def register_task(client_exe_path, msg):
        request = WorkRequest(thread_work, (client_exe_path, msg), {}, callback=thread_work_cb,
                    exc_callback=handle_exception)
        thread_pool.putRequest(request)

    def get_client_exe_paths(client_name):
        client_exe_paths = []
        for client_id in CONF.clients.clients_list:
            client_conf = getattr(CONF, client_id)
            client_n = client_conf.name
            if client_n == client_name:
                client_exe_paths.append(client_conf.exe_path)
        return client_exe_paths

    while True:
        events = dict(poller.poll(2000))
        if events.get(socket0) == zmq.POLLIN:
            # Deal with message
            _msg = socket0.recv()

            print("Received request: [%s]\n" % (_msg))
            client_name = _msg.split()[0]
            msg = _msg[len(client_name):].strip()
            exe_paths = get_client_exe_paths(client_name)
            logger.debug("Client name \"%s\" message: %s" %(client_name, msg))
            logger.debug("Client exe path %s" % exe_paths)
            if exe_paths is not None:
                for exe_path in exe_paths:
                    register_task(exe_path, msg)
                #send reply back to client
                socket0.send("OK")
            else:
                logger.debug("Can not execute client, because there have none clients in config file(%s), please check" % str(CONF.clients.clients_list))
                socket0.send('FAILED')

        if events.get(socket_exit) == zmq.POLLIN:
            cmd = socket_exit.recv()
            logger.debug('%s CMD %s' % (worker_id, cmd))
            sys.stdout.flush()
            if cmd == 'EXIT':
                break
            elif cmd == 'CONFIG_CHANGED':
                reload_config()
            else:
                pass
    thread_pool.close()
    thread_pool.join()
    socket0.close()
    socket_exit.close()