Python Pool.apply_async方法代码示例

# 需要导入模块: from multiprocess import Pool [as 别名]
# 或者: from multiprocess.Pool import apply_async [as 别名]
 def inner(*args):
     pool = Pool(processes=1)
     res = pool.apply_async(f,args)
     try:
         v = res.get(timeout=sec)
     except Exception as inst:
         print(inst)
         v = None
     finally:
         pool.terminate()
         return v

# 需要导入模块: from multiprocess import Pool [as 别名]
# 或者: from multiprocess.Pool import apply_async [as 别名]
def download_image_thread(location_q, image_q, MAX_DL_THREADS=10):
    print("Running Download Image Thread.")

    max_processes = MAX_DL_THREADS
    print("Creating a thread pool of size {} for downloading images...".format(max_processes))
    pool = Pool(processes=max_processes)
    # Allow us to have n processes runnning, and n processes scheduled to run
    # TODO: Manager is not necessary here, but is used to get around the fact
    # that thread-safe objects cannot be passed by reference, they must be
    # inheretence. A more lightweight solution should be found
    workers = Manager().Semaphore(max_processes*2)

    def async_download(location):
        image = download_image(location)
        image_q.put((location, image), True)
        workers.release()

    while True:
        location = location_q.get(True)
        workers.acquire()
        pool.apply_async(async_download, (location,))

# 需要导入模块: from multiprocess import Pool [as 别名]
# 或者: from multiprocess.Pool import apply_async [as 别名]
    def prime_calculate(self):
        break_points = []  # List that will have start and stopping points
        for i in range(cores):  # Creates start and stopping points based on length of range_finish
            break_points.append(
                {"start": int(math.ceil(((self.maximum_prime + 1) + 0.0) / cores * i)),
                 "stop": int(math.ceil(((self.maximum_prime + 1) + 0.0) / cores * (i + 1)))})

        p = Pool(cores)  # Number of processes to create.
        for i in break_points:  # Cycles though the breakpoints list created above.
            a = p.apply_async(self.prime_calculator, kwds=i, args=tuple(),
                              callback=self.update_num)  # This will start the separate processes.
        p.close()  # Prevents any more processes being started
        p.join()  # Waits for worker process to end

# 需要导入模块: from multiprocess import Pool [as 别名]
# 或者: from multiprocess.Pool import apply_async [as 别名]
class ProcessPoolExecutor(Executor):
    """Process Pool Executor"""
    def __init__(self):
        super(ProcessPoolExecutor, self).__init__()
        import os
        from multiprocess import Pool
        self.pool = Pool(os.cpu_count() or 1)

    def submit(self, func, *args, **kwargs):
        from concurrent.futures import Future
        fut = Future()
        self.tasks[fut] = self.pool.apply_async(
            func, args, kwargs, fut.set_result, fut.set_exception
        )
        fut.add_done_callback(self.tasks.pop)
        return fut

    def shutdown(self, wait=True):
        super(ProcessPoolExecutor, self).shutdown(wait)
        self.pool.terminate()
        self.pool.join()

# 需要导入模块: from multiprocess import Pool [as 别名]
# 或者: from multiprocess.Pool import apply_async [as 别名]

#.........这里部分代码省略.........
        else:
            self.operation_table[dataset_id] = [operation]

        # Add data block to scheduler
        if prio == Priority.high:
            if dataset_id not in self.high_access:
                self.high_access.append(dataset_id)
                if dataset_id in self.normal_access:
                    self.normal_access.remove(dataset_id)

        elif prio == Priority.normal:
            if dataset_id not in self.normal_access and dataset_id not in self.high_access:
                self.normal_access.append(dataset_id)

    def _run_queue(self, dataset_id, debug=False):
        '''
        Run the queue of operations for a given dataset
        '''
        # Create data queue and a storage reading process
        if debug:
            print('~ Request data blocks from reading process')

        data_queue = self.manager.Queue()
        self.storage.read_data(dataset_id, data_queue)

        # Amount of operations
        operations = len(self.operation_table[dataset_id])

        # Amount of data-blocks
        data_blocks = self.storage.get_size(dataset_id)

        # Create a result list to each operation
        results = []
        for i in range(operations):
            results.append([])

        # Execute map-operation on the data queue
        for i in range(data_blocks):
            try:
                # Fetch data block from data queue
                data_block = data_queue.get(timeout=3)

                print('- Performing operations on block: ' + str(i) + ', dataset: ' + dataset_id)

                # Perform the operations on fetched data block
                op_index = 0
                for operation in self.operation_table[dataset_id]:
                    if debug:
                        print('~ Performing map operation (' + str(operation) + ') on block ' + str(i))

                    results[op_index].append(operation.map(data_block))
                    op_index += 1
            except:
                print('! Timeouted waiting for data')

        # Execute the reduce-operation
        op_index = 0
        for operation in self.operation_table[dataset_id]:
            if debug:
                print('~ Performing reduce operation (' + str(operation) + ')')

            operation.reduce(results[op_index])
            op_index += 1

        # Clear the operation table for this block
        if debug:
            print('~ Clearing operations for '+ str(dataset_id))

        self.operation_table[dataset_id] = []

        # Remove the operation meta data for the dataset
        if operations > 0:
            if debug:
                print('~ Removing the dataset '+ str(dataset_id) + ' from operation table')

            del self.operation_table[dataset_id]

    def schedule(self, debug=False):
        '''
        Schedule the queued operations
        '''
        if debug:
            print('~ Initiating reading process')

        reading_process = Process(target=self.storage.reader)
        reading_process.start()

        while True:
            if debug:
                print()
                print('/ High priority queue is ' + str(self.high_access))
                print('/ Normal priority queue is ' + str(self.normal_access))
                print()

            if self.high_access:
                self._pool.apply_async(self._run_queue(self.high_access.pop(0)))
            elif self.normal_access:
                self._pool.apply_async(self._run_queue(self.normal_access.pop(0)))
            else:
                time.sleep(0.5)

# 需要导入模块: from multiprocess import Pool [as 别名]
# 或者: from multiprocess.Pool import apply_async [as 别名]
def test():
    print('cpuCount() = %d\n' % cpuCount())
    
    #
    # Create pool
    #
    
    PROCESSES = 4
    print('Creating pool with %d processes\n' % PROCESSES)
    pool = Pool(PROCESSES)    

    #
    # Tests
    #

    TASKS = [(mul, (i, 7)) for i in range(10)] + \
            [(plus, (i, 8)) for i in range(10)]

    results = [pool.apply_async(calculate, t) for t in TASKS]
    imap_it = pool.imap(calculatestar, TASKS)
    imap_unordered_it = pool.imap_unordered(calculatestar, TASKS)

    print('Ordered results using pool.apply_async():')
    for r in results:
        print('\t', r.get())
    print()

    print('Ordered results using pool.imap():')
    for x in imap_it:
        print('\t', x)
    print()

    print('Unordered results using pool.imap_unordered():')
    for x in imap_unordered_it:
        print('\t', x)
    print()

    print('Ordered results using pool.map() --- will block till complete:')
    for x in pool.map(calculatestar, TASKS):
        print('\t', x)
    print()

    #
    # Simple benchmarks
    #

    N = 100000
    print('def pow3(x): return x**3')
    
    t = time.time()
    A = list(map(pow3, xrange(N)))
    print('\tmap(pow3, xrange(%d)):\n\t\t%s seconds' % \
          (N, time.time() - t))
    
    t = time.time()
    B = pool.map(pow3, xrange(N))
    print('\tpool.map(pow3, xrange(%d)):\n\t\t%s seconds' % \
          (N, time.time() - t))

    t = time.time()
    C = list(pool.imap(pow3, xrange(N), chunksize=N//8))
    print('\tlist(pool.imap(pow3, xrange(%d), chunksize=%d)):\n\t\t%s' \
          ' seconds' % (N, N//8, time.time() - t))
    
    assert A == B == C, (len(A), len(B), len(C))
    print()
    
    L = [None] * 1000000
    print('def noop(x): pass')
    print('L = [None] * 1000000')
    
    t = time.time()
    A = list(map(noop, L))
    print('\tmap(noop, L):\n\t\t%s seconds' % \
          (time.time() - t))
    
    t = time.time()
    B = pool.map(noop, L)
    print('\tpool.map(noop, L):\n\t\t%s seconds' % \
          (time.time() - t))

    t = time.time()
    C = list(pool.imap(noop, L, chunksize=len(L)//8))
    print('\tlist(pool.imap(noop, L, chunksize=%d)):\n\t\t%s seconds' % \
          (len(L)//8, time.time() - t))

    assert A == B == C, (len(A), len(B), len(C))
    print()

    del A, B, C, L

    #
    # Test error handling
    #

    print('Testing error handling:')

    try:
        print(pool.apply(f, (5,)))
    except ZeroDivisionError:
#.........这里部分代码省略.........

# 需要导入模块: from multiprocess import Pool [as 别名]
# 或者: from multiprocess.Pool import apply_async [as 别名]
class GroupCheckerGui(BaseWidget):

    def __init__(self):
        super(GroupCheckerGui, self).__init__('Group Checker')

        self._group_name = ControlText('Group Name', CONFIG['group_name'])
        self._group_name.enabled = False
        self._allowed_tags = UnicodeControlList('Allowed Tags',
                                               plusFunction=self.__add_tag_action,
                                               minusFunction=self.__remove_tag_action)
        self.allowed_tags = GuiList(CONFIG['white_filters']['SubstringFilter']['substrings'],
                                    self._allowed_tags)

        self._allowed_ids = ControlList('Allowed Ids',
                                        plusFunction=self.__add_id_action,
                                        minusFunction=self.__remove_id_action)
        self.allowed_ids = GuiList(CONFIG['white_filters']['SignerFilter']['ids'], self._allowed_ids)

        self._bad_posts = ControlCheckBoxList('Bad posts')
        self._bad_posts._form.listWidget.itemDoubleClicked.connect(self.__show_link_action)

        self._remove_button = ControlButton('Remove')
        self._remove_button.value = self.__remove_action

        self._show_button = ControlButton('Show bad posts')
        self._show_button.value = self.__show_bad_post_action

        self.pool = Pool(processes=1)
        self.bad_posts = []

        self._formset = [('', '_group_name', ''),
                         ('', '_allowed_tags', '_allowed_ids', ''),
                         '',
                         ('', '_bad_posts', ''),
                         ('', '_remove_button', '_show_button', ''),
                         '']

    def __add_tag_action(self):
        win = PopUpGetText('tag', self.allowed_tags)
        win.show()

    def __remove_tag_action(self):
        self.allowed_tags.remove(self._allowed_tags.mouseSelectedRowIndex)

    def __add_id_action(self):
        win = PopUpGetText('id', self.allowed_ids)
        win.show()

    def __remove_id_action(self):
        self.allowed_ids.remove(self._allowed_ids.mouseSelectedRowIndex)

    def __show_bad_post_action(self):
        def callback(posts):
            self.bad_posts = posts
            self._bad_posts.value = [(GroupBot.get_link_from_post(post, CONFIG['group_name']), True) for post in posts]
            self._show_button.enabled = True

        def run_bot():
            bot = create_bot_from_config()
            return bot.get_bad_posts()

        self._show_button.enabled = False
        self.pool.apply_async(run_bot, callback=callback)

    def __show_link_action(self, link):
        webbrowser.open(link.text())

    def __remove_action(self):
        checked_posts = [self.bad_posts[idx] for idx in self._bad_posts.checkedIndexes]
        bot = create_bot_from_config()
        bot.remove_posts(checked_posts)