Python Pool.map方法代码示例

# 需要导入模块: from multiprocessing import Pool [as 别名]
# 或者: from multiprocessing.Pool import map [as 别名]
def main():
    start()
    for url in seen_urls:
        print(url)
    # print(seen_urls)
    p = Pool(4)
    p.map(get_data, next_urls)

# 需要导入模块: from multiprocessing import Pool [as 别名]
# 或者: from multiprocessing.Pool import map [as 别名]
def train_word2id():
    """把训练集的所有词转成对应的id。"""
    time0 = time.time()
    print('Processing train data.')
    df_train = pd.read_csv('../raw_data/question_train_set.txt', sep='\t', usecols=[0, 2, 4],
                           names=['question_id', 'word_title', 'word_content'], dtype={'question_id': object})
    print('training question number %d ' % len(df_train))
    # 没有 content 的问题用 title 来替换
    na_content_indexs = list()
    for i in tqdm(xrange(len(df_train))):
        word_content = df_train.word_content.values[i]
        if type(word_content) is float:
            na_content_indexs.append(i)
    print('There are %d train questions without content.' % len(na_content_indexs))
    for na_index in tqdm(na_content_indexs):
        df_train.at[na_index, 'word_content'] = df_train.at[na_index, 'word_title']
    # 没有 title 的问题， 丢弃
    na_title_indexs = list()
    for i in xrange(len(df_train)):
        word_title = df_train.word_title.values[i]
        if type(word_title) is float:
            na_title_indexs.append(i)
    print('There are %d train questions without title.' % len(na_title_indexs))
    df_train = df_train.drop(na_title_indexs)
    print('After dropping, training question number(should be 2999952) = %d' % len(df_train))
    # 转为 id 形式
    p = Pool()
    train_title = np.asarray(p.map(get_id4words, df_train.word_title.values))
    np.save('../data/wd_train_title.npy', train_title)
    train_content = np.asarray(p.map(get_id4words, df_train.word_content.values))
    np.save('../data/wd_train_content.npy', train_content)
    p.close()
    p.join()
    print('Finished changing the training words to ids. Costed time %g s' % (time.time() - time0))

# 需要导入模块: from multiprocessing import Pool [as 别名]
# 或者: from multiprocessing.Pool import map [as 别名]
def fetch_imagery(image_locations, local_dir):
    pool = Pool(cpu_count())
    tupled = [(loc[0], loc[1], local_dir) for loc in image_locations]
    try:
        pool.map(fetch_imagery_uncurried, tupled)
    finally:
        pool.close()

# 需要导入模块: from multiprocessing import Pool [as 别名]
# 或者: from multiprocessing.Pool import map [as 别名]
def lvl2_data(obsid_list):

    p = Pool(4)
    if len(glob.glob(obs+'/*/*evt2.fits')) == 1:
        print('Found obsID repro directory')
    else:
        p.map(repro_worker, obsid_list)

# 需要导入模块: from multiprocessing import Pool [as 别名]
# 或者: from multiprocessing.Pool import map [as 别名]
def updateTranslation(args):
    # Get map that contains (besides other stuff)
    #  the crowdin ID for a given file
    translationFilemap = getTranslationFilemapCache(args.language, args.force_filemap_update)

    # Collect valid downloadable files for parallel processing
    fileinfos = []
    for filename, fileinfo in translationFilemap.items():
        filepath = os.path.join("cache", args.language, fileinfo["path"])
        # Create dir if not exists
        try: os.makedirs(os.path.dirname(filepath))
        except OSError as exc:
            if exc.errno == errno.EEXIST:
                pass
            else:
                raise
        fileid = fileinfo["id"]
        fileinfos.append((fileid, filepath))
    # Curry the function with the language
    performDownload = functools.partial(performPOTDownload, args.language)
    # Perform parallel download
    if args.num_processes > 1:
        pool = Pool(args.num_processes)
        pool.map(performDownload, fileinfos)
    else:
        for t in fileinfos:
            performDownload(t)
    #Set download timestamp
    timestamp = datetime.datetime.now().strftime("%y-%m-%d %H:%M:%S")
    with open("lastdownload.txt", "w") as outfile:  
        outfile.write(timestamp)

# 需要导入模块: from multiprocessing import Pool [as 别名]
# 或者: from multiprocessing.Pool import map [as 别名]
def main(argv):
    # help option
    try:
        opts, args = getopt.getopt(argv, 'h')
        for opt, arg in opts:
            if opt == '-h':
                print('''args: 3 pickle files folder_of_data number_of_cores:
                clf_alc.p clf_fpa.p clf_fpl.p tweets_folder 8''')
                sys.exit(2)
    except getopt.GetoptError:
        print('-h for help')
        sys.exit(2)

    # run prediction
    s_clf_alc, s_clf_fpa, s_clf_fpl, folder, out_dir, cores = tuple(argv)
    cores = int(cores)
    clf_alc = pickle.load(open(s_clf_alc, 'rb'))
    clf_fpa = pickle.load(open(s_clf_fpa, 'rb'))
    clf_fpl = pickle.load(open(s_clf_fpl, 'rb'))
    clf = PredictionTransformer(clf_alc, clf_fpa, clf_fpl)

    # parallel
    p = Pool(cores)
    dirs = [(clf, out_dir, folder + '/' + f) for f in os.listdir(folder)]
    p.map(predict, dirs)

# 需要导入模块: from multiprocessing import Pool [as 别名]
# 或者: from multiprocessing.Pool import map [as 别名]
def directoryProcessor(dir, parallel):
    files = os.listdir(dir)
    p = Pool(parallel)
    tasks = []
    for i in range(len(files)):
        tasks.append((J(dir, files[i]), i))
    p.map(fileProcessor, tasks)

# 需要导入模块: from multiprocessing import Pool [as 别名]
# 或者: from multiprocessing.Pool import map [as 别名]
def mutliprocess_start(process_num=20, limit=1000):
    from multiprocessing import Pool as JPool  # 多进程
    from multiprocessing import cpu_count
    pool = JPool(process_num * cpu_count())
    pool.map(manual_start, (i for i in range(MAX_TASK_NUMBER / 5)))
    pool.close()
    pool.join()

# 需要导入模块: from multiprocessing import Pool [as 别名]
# 或者: from multiprocessing.Pool import map [as 别名]
    def extract(self, parallel = True, metrics = True):
        """Extracts data from the netcdf files into a more useful format."""

        self.pyhspf_names = ['rain',
                             'temperature',
                             'humidity',
                             'solar',
                             'snowdepth',
                             'evaporation',
                             'wind',
                             ]
        
        if parallel:
            pool = Pool(len(self.pyhspf_names))
            pool.map(self.organize, self.pyhspf_names)
            pool.close()

        else: 
            for v in pyhspf_names: self.organize(v)
                  
        # re-organize into NRCM classes at each grid point

        stationdata = '{}/gridpoints'.format(self.destination)
        if not os.path.isdir(stationdata): os.mkdir(stationdata)

        if parallel:
            pool = Pool(cpu_count())
            x = pool.map(self.extract_gridpoints, self.names)
            pool.close()

        else:
            for n in self.names: self.extract_gridpoints(n)

        if metrics: self.extract_metrics()

# 需要导入模块: from multiprocessing import Pool [as 别名]
# 或者: from multiprocessing.Pool import map [as 别名]
    def _do_multiprocess_clean(self):
        params = []
        for package_name in self._package_map:
            params.append((self._npm, self._package_map[package_name]))

        pool = Pool(cpu_count())
        pool.map(clean_dependencies, params)

# 需要导入模块: from multiprocessing import Pool [as 别名]
# 或者: from multiprocessing.Pool import map [as 别名]
def start_crawlers(connector_class, processes=1, debug=False):
    """

    Start spider processes

    :param processes:
    :param connector_class:
    :param debug:
    :return:
    """
    spiders_classes = get_spiders_classes()

    if processes == 0:
        connector = connector_class()
        with _get_lock('ALL') as acquired:
            if acquired:
                crawl(spiders_classes, connector, debug)
            else:
                print("Crawl process of 'ALL' already running")
            return

    # split list in x list of processes count elements
    spider_classes_chunks = [spiders_classes[x:x + processes] for x in range(0, len(spiders_classes), processes)]

    # Start one cycle of processes by chunk
    for spider_classes_chunk in spider_classes_chunks:
        process_params_chunk = [(spider_class, connector_class, debug) for spider_class in spider_classes_chunk]
        p = Pool(len(process_params_chunk))
        p.map(start_crawl_process, process_params_chunk)

# 需要导入模块: from multiprocessing import Pool [as 别名]
# 或者: from multiprocessing.Pool import map [as 别名]
def main():
  """Take in the arguments"""

  parser = argparse.ArgumentParser(description='Run the Benchmark script')
  parser.add_argument('host', action="store", help='HostName')
  parser.add_argument('number_iterations', action="store", type=int, help='Number of iterations')
  parser.add_argument('number_processes', action="store", type=int, help='Number of processes')

  result = parser.parse_args()

  global SITE_HOST
  SITE_HOST = result.host
  zidx_list = range(result.number_iterations)
  random.shuffle(zidx_list)
  post_list = []
  for zidx in zidx_list:
    post_list.append(generateURL(zidx))
  from multiprocessing import Pool
  pool = Pool(result.number_processes)
  start = time.time()
  pool.map(postURLHelper, post_list)
  print time.time() - start

  # KL TODO insert get data here
  getURL(generateURL2(zidx_list[0]))
  getURL(generateURL2(zidx_list[0]))

# 需要导入模块: from multiprocessing import Pool [as 别名]
# 或者: from multiprocessing.Pool import map [as 别名]
def main():
    from multiprocessing import Pool
    p = Pool(20)
    begin = time.perf_counter()
    p.map(run_client, [100] * 5)
    end = time.perf_counter()
    print((end - begin) * 1000)

# 需要导入模块: from multiprocessing import Pool [as 别名]
# 或者: from multiprocessing.Pool import map [as 别名]
    def test_runMediaTest(self,server,user,password):
        #local use
        #/Users/mgarthwaite/Dropbox/CAH_Recorded
        #/zoidberg/CI/CAH_Recorded

        pictureList, videoList = self.appendList("/zoidberg/CI/CAH_Recorded")

        if len(pictureList) > len(videoList):
             listCount = len(pictureList)
        else:
             listCount = len(videoList)

        func = partial(runTest, server, user, password, pictureList, videoList)
        pool = Pool(processes=8)
        pool.map(func,range(0,listCount))
        pool.close()
        pool.join()


        logFile = self.aggregateLogs(listCount)
        logFile.seek(0)
        data = logFile.read()
        count = data.count("fail: 1")

        if count == 0:
            print "NO FAILED TESTS. GREEN"
            sys.exit(0)
        elif count < listCount and count > 0:
            print "SOME FAILED TESTS. YELLOW"
            sys.exit(0)
        else: #count >= listCount
            print "ALL TESTS FAILED. RED."
            sys.exit(1)

# 需要导入模块: from multiprocessing import Pool [as 别名]
# 或者: from multiprocessing.Pool import map [as 别名]
def reduce(self, params):
    from os import listdir
    files = [f for f in listdir(".") 
             if f.startswith(params.prefix) and f.endswith(".root")]
    
    def get_period(s): return s.split("-")[1].split("_")[-1]
    
    by_subperiod = {}
    for f in files:
        by_subperiod.setdefault(get_period(f), []).append(f)
        
    from pprint import pprint
    pprint(by_subperiod)
        
    from multiprocessing import Pool, cpu_count
    pool = Pool(cpu_count())
    
    pool.map(mp_merge, [("period%s.root" % p, files) for p, files in by_subperiod.iteritems()])
    
    by_period = {}
    for p, files in sorted(by_subperiod.iteritems()):
        by_period.setdefault(p[0], []).append("period%s.root" % p)
    
    pprint(by_period)
    
    pool.map(mp_merge, [("period%s.root" % p, files) for p, files in by_period.iteritems()])
    
    from hmerge import merge_files
    merge_files("all.root", ["period%s.root" % p for p in by_period])
    
    print "Done."