Python pool.map函数代码示例

 def _push(self, src, dst):
     """
     Push src to dst on the remote.
     """
     force = False
     if src.startswith('+'):
         src = src[1:]
         force = True
     present = [self._refs[name][1] for name in self._refs]
     present.extend(self._pushed.values())
     # before updating the ref, write all objects that are referenced
     objects = git_list_objects(src, present)
     try:
         # upload objects in parallel
         pool = multiprocessing.pool.ThreadPool(processes=self._processes)
         pool.map(Binder(self, '_put_object'), objects)
     except Exception:
         self._fatal('exception while writing objects')
     sha = git_ref_value(src)
     error = self._write_ref(sha, dst, force)
     if error is None:
         self._write('ok %s' % dst)
         self._pushed[dst] = sha
     else:
         self._write('error %s %s' % (dst, error))

def parallel_compile(self, sources, output_dir=None, macros=None,
                     include_dirs=None, debug=0, extra_preargs=None,
                     extra_postargs=None, depends=None):
    """New compile function that we monkey patch into the existing compiler instance.
    """
    import multiprocessing.pool

    # Copied from the regular compile function
    macros, objects, extra_postargs, pp_opts, build = \
            self._setup_compile(output_dir, macros, include_dirs, sources,
                                depends, extra_postargs)
    cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)

    def _single_compile(obj):
        try:
            src, ext = build[obj]
        except KeyError:
            return
        self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)

    # Set by fix_compiler
    global glob_use_njobs
    if glob_use_njobs == 1:
        # This is equivalent to regular compile function
        for obj in objects:
            _single_compile(obj)
    else:
        # Use ThreadPool, rather than Pool, since the objects are picklable.
        pool = multiprocessing.pool.ThreadPool(glob_use_njobs)
        pool.map(_single_compile, objects)
        pool.close()
        pool.join()

    # Return *all* object filenames, not just the ones we just built.
    return objects

def run_all_intersections(write_anomaly, incomplete, intersections, multi_model, smooth):
    print "Running all on", os.getpid()
    start_time = time.time()

    if incomplete:
        key = '_id'
        query = [
            {'$match': {'anomaly': {'$exists': False}}},
            {'$group': {'_id': '$site_no'}}
        ]
        if intersections != '':
            query[0]['$match']['site_no'] = {'$in': intersections.split(',')}
        locations = list(readings_collection.aggregate(query))

    else:
        key = 'intersection_number'
        if intersections != '':
            query = {key: {'$in': intersections.split(',')}}
        else:
            query = {key:  {'$regex': '3\d\d\d'}}
        locations = list(locations_collection.find(query))
    gen = [(str(l[key]), write_anomaly, incomplete, False, multi_model, smooth) for l in locations]
    pool = Pool(8, maxtasksperchild=1)
    pool.map(run_single_intersection, gen)
    print("TOTAL TIME: --- %s seconds ---" % (time.time() - start_time))

  def build_packages(self):
      """Build all the Spinnaker packages."""
      all_subsystems = []
      all_subsystems.extend(SUBSYSTEM_LIST)
      all_subsystems.extend(ADDITIONAL_SUBSYSTEMS)

      if self.__options.build:
        # Build in parallel using half available cores
        # to keep load in check.
        weighted_processes = self.__options.cpu_ratio * multiprocessing.cpu_count()
        pool = multiprocessing.pool.ThreadPool(
            processes=int(max(1, weighted_processes)))
        pool.map(self.__do_build, all_subsystems)

      if self.__build_failures:
        if set(self.__build_failures).intersection(set(SUBSYSTEM_LIST)):
          raise RuntimeError('Builds failed for {0!r}'.format(
            self.__build_failures))
        else:
          print 'Ignoring errors on optional subsystems {0!r}'.format(
              self.__build_failures)

      if self.__options.nebula:
        return

      wait_on = set(all_subsystems).difference(set(self.__build_failures))
      pool = multiprocessing.pool.ThreadPool(processes=len(wait_on))
      print 'Copying packages...'
      pool.map(self.__do_copy, wait_on)
      return

def main():
    parser = argparse.ArgumentParser(
        description="A simple tool to backup your Bitbucket repositories",
    )
    
    parser.add_argument('username', type=str, help='Username')
    parser.add_argument('password', type=str, help='Password')
    parser.add_argument('backupdir', type=str, 
                        help='The target backup directory')
    
    args = parser.parse_args()
    
    bitbucket = Bitbucket(args.username, args.password)
    repos = list(bitbucket.get_repositories())
    random.shuffle(repos)

    pool = multiprocessing.pool.ThreadPool(20)
    pool.map(lambda x: x.backup(args.backupdir), repos)

    failed = 0
    for repo in repos:
        if repo.failed is None:
            continue
        failed += 1
        print 'WARNING: the following repositories failed to update:'
        print repo.name
        print repo.output
        print repo.failed
    if failed:
        sys.exit(2)

def get_item_by_url(urls):

    pool =mul.Pool()

    t1 =clock()
    pool.map(get_item_info,urls)
    t2=clock()
    print 'time\t'+str(t2-t1)
    print 'total count\t'+str(count)

	def run_master(self):

		logging.info('Creating a pool of ' + str(self.num_processes) + ' subprocess workers.')
		# create a pool of processes.
		pool = Pool(processes=self.num_processes,)

		# apply map on the chunks in parallel.
		regions = pool.map(self.apply_map, range(0, self.num_processes))

		# do the intermediate grouping step on each chunks in parallel.
		inters = pool.map(self.apply_intermediate, range(0, self.num_processes))

def scan_all():
    items = [(name, addr)
             for name, endpoints in config['endpoints'].items()
             for addr in endpoints]
    pool.map(scan_one, items)

    info()

    if 'verbose' in sys.argv:
        import pprint;
        pprint.pprint(dict(active))
        pprint.pprint(dict(inactive))

    header = "".join([
        "name".center(29),
        "active".rjust(8),
        "inactive".rjust(9),
        "percent".rjust(9),
        "reason".center(32),
    ])
    info()
    info(header + "\n")
    info("-" * len(header) + "\n")

    active_n_total, inactive_n_total = 0, 0
    for name in sorted(config['endpoints']):
        active_n = len(active[name])
        inactive_n = len(inactive[name])

        active_n_total += active_n
        inactive_n_total += inactive_n

        total = active_n + inactive_n

        percent = ""
        if total:
            percent = "%%%0.1f" % (100 * float(active_n) / total)

        reasons = set([reason for _, reason in inactive[name]])

        info(name.rjust(29))
        info(str(active_n).rjust(8))
        info(str(inactive_n).rjust(9))
        info(percent.rjust(9))
        info(", ".join(reasons).rjust(32) + "\n")

    info("-" * len(header) + "\n")

    info("  total active:   %i\n" % active_n_total)
    info("total inactive:   %i\n" % inactive_n_total)
    value = 100 * float(active_n_total) / (active_n_total + inactive_n_total)
    info("percent active: %%%0.1f\n" % value)
    return value

def _ConvertToWebP(webp_binary, png_files):
  pool = multiprocessing.pool.ThreadPool(10)
  def convert_image(png_path):
    root = os.path.splitext(png_path)[0]
    webp_path = root + '.webp'
    args = [webp_binary, png_path] + _PNG_TO_WEBP_ARGS + [webp_path]
    subprocess.check_call(args)
    os.remove(png_path)
  # Android requires pngs for 9-patch images.
  pool.map(convert_image, [f for f in png_files if not f.endswith('.9.png')])
  pool.close()
  pool.join()

  def build_jars(self):
    """Build the Spinnaker packages as jars
    """
    subsystems = ['halyard']

    if self.__options.do_jar_build:
      weighted_processes = self.__options.cpu_ratio * multiprocessing.cpu_count()
      pool = multiprocessing.pool.ThreadPool(
        processes=int(max(1, weighted_processes)))
      pool.map(self.__do_jar_build, subsystems)

    self.__check_build_failures(subsystems)

  def build_container_images(self):
    """Build the Spinnaker packages as container images.
    """
    subsystems = [comp for comp in SUBSYSTEM_LIST if comp != 'spinnaker']
    subsystems.append('spinnaker-monitoring')

    if self.__options.container_builder:
      weighted_processes = self.__options.cpu_ratio * multiprocessing.cpu_count()
      pool = multiprocessing.pool.ThreadPool(
        processes=int(max(1, weighted_processes)))
      pool.map(self.__do_container_build, subsystems)

    self.__check_build_failures(subsystems)

	def run_program(self):

		logging.info('Running the framework...')
		# Fixing the start time.
		start_time = time.time()

		"""
		Create a pool of processes. The number of processes
		is equal to the number of files. One process takes care of one file.
		"""
		pool = MyMRPool(len(self.files))

		logging.info('The initial number of running processes is ' + str(len(self.files)) + '.')


		"""
		Apply call_map_reduce on all files in parallel. All files 
		will be partitioned/mapped/shuffled individually.
		"""
		apply_map_reduces = pool.map(self.call_map_reduce, self.files)


		self.shuffle()

		"""
		At this point we have bunch of inter-shuffled files.
		We can reduce them in parallel.	
		"""
		reduces = pool.map(self.apply_reduce, range(0, self.num_processes))

		"""
		At this point we have bunch of reduced files so we can 
		merge all reduce files into one final file.	
		"""
		self.merge_reduce_results()

		"""
		Finilizing the framework execution.	
		"""
		self.__finalize_program()

		logging.info('The program is successfully finished.')

		"""
		Fixing the end time. We use this for calculating
		the total execution time of the framework.
		"""
		end_time = time.time()

		logging.info('The total execution time is: ' + str(end_time - start_time))

def executeOperatorSequence(operator, kwargsUpdated, parallel):   
    outputPathPattern = ''
    inputPathPattern = ''
    for key, value in kwargsUpdated.iteritems():
        arg = str(value)
        if '*' in arg:

            if operator.get_targets().count(key) == 1:
                outputPathPattern = arg
                outputKey = key
            else: 
                # get file of path in arg with Unix style pathname pattern expansion
                fileList = glob.glob(arg)
                if not fileList:
                    log.error("%s: Could not find any files for input pattern %s in Slot %s" % operator.name, outputPathPattern, outputKey)
                
                inputFileList = fileList
                inputPathPattern = arg
                inputKey = key

    if outputPathPattern == '' or inputPathPattern == '':
        log.error("If two file patterns (paths with '*' are used, one must be an argument for a non-target parameter and one in target parameter")
            
    pre, post = inputPathPattern.split('*')
    outputFileList = []
    
    for fil in inputFileList:
        tmp = str(fil).replace(pre, '')
        tmp = str(tmp).replace(post, '')  
        outputFileList.append(outputPathPattern.replace('*', tmp))
    
    args_list = []
    for j in range(len(inputFileList)):
        kwargs_new = OrderedDict(kwargsUpdated)
        kwargs_new[inputKey] = inputFileList[j]
        kwargs_new[outputKey] = outputFileList[j]
        args_new = list(kwargs_new.values())
        args_new.append(operator)            
        args_list.append(args_new)
        
    if parallel:
        # multiprocessing
        num_of_workers = multiprocessing.cpu_count()
        pool = multiprocessing.Pool(num_of_workers - 1)
        # blocks until finished
        pool.map(callWrapper, args_list)
    else:
        for args in args_list:
            callWrapper(args)
    return outputPathPattern

def _ConvertToWebP(webp_binary, png_files):
  pool = multiprocessing.pool.ThreadPool(10)
  def convert_image(png_path):
    root = os.path.splitext(png_path)[0]
    webp_path = root + '.webp'
    args = [webp_binary, png_path, '-mt', '-quiet', '-m', '6', '-q', '100',
        '-lossless', '-o', webp_path]
    subprocess.check_call(args)
    os.remove(png_path)

  pool.map(convert_image, [f for f in png_files
                           if not _PNG_WEBP_BLACKLIST_PATTERN.match(f)])
  pool.close()
  pool.join()

def process_images():
    """Process all images in parallel.

    Like app.process_images, use a process pool for convenience. The last
    three steps of the problem (cropping and saving) are also parallelized.
    This cannot be done using multiprocessing.Pool because it daemonizes its
    children processes, and they in turn cannot have children of their own.

    Use custom Pool and Process subclasses that ensure the children are not
    daemonized.
    """
    pool = NoDaemonPool()  # use cpu_count() processes
    pool.map(process_image, image_paths())
    pool.close()
    pool.join()