Python multiprocessing.cpu_count方法代码示例

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import cpu_count [as 别名]
def get_graph_stats(graph_obj_handle, prop='degrees'):
    # if prop == 'degrees':
    num_cores = multiprocessing.cpu_count()
    inputs = [int(i*len(graph_obj_handle)/num_cores) for i in range(num_cores)] + [len(graph_obj_handle)]
    res = Parallel(n_jobs=num_cores)(delayed(get_values)(graph_obj_handle, inputs[i], inputs[i+1], prop) for i in range(num_cores))

    stat_dict = {}

    if 'degrees' in prop:
        stat_dict['degrees'] = list(set([d for core_res in res for file_res in core_res for d in file_res['degrees']]))
    if 'edge_labels' in prop:
        stat_dict['edge_labels'] = list(set([d for core_res in res for file_res in core_res for d in file_res['edge_labels']]))
    if 'target_mean' in prop or 'target_std' in prop:
        param = np.array([file_res['params'] for core_res in res for file_res in core_res])
    if 'target_mean' in prop:
        stat_dict['target_mean'] = np.mean(param, axis=0)
    if 'target_std' in prop:
        stat_dict['target_std'] = np.std(param, axis=0)

    return stat_dict 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import cpu_count [as 别名]
def get_cpuusage(filename,field_values,which_dict):
    cpuusage_file = open(os.path.join(homepath,datadir,filename))
    lines = cpuusage_file.read().split("\n")
    cpu_dict={}
    cpu_count = multiprocessing.cpu_count()
    for i in range(0,cpu_count):
        cpucore = "cpu"+str(i)
        cpu_dict[cpucore] = {}
    for eachline in lines:
        tokens_split = eachline.split("=")
        if(len(tokens_split) == 1):
            continue
        cpucoresplit = tokens_split[0].split("$")
        cpu_dict[cpucoresplit[0]][cpucoresplit[1]] = float(tokens_split[1])
    totalresult = 0
    for i in range(0,cpu_count):
        cpucore = "cpu"+str(i)
        which_dict["cpu_usage"] = cpu_dict
        Total = cpu_dict[cpucore]["user"] + cpu_dict[cpucore]["nice"] + cpu_dict[cpucore]["system"] + cpu_dict[cpucore]["idle"] + cpu_dict[cpucore]["iowait"] + cpu_dict[cpucore]["irq"] + cpu_dict[cpucore]["softirq"]
        idle = cpu_dict[cpucore]["idle"] + cpu_dict[cpucore]["iowait"]
        field_values[0] = "CPU"
        result = 1 - round(float(idle/Total),4)
        totalresult += float(result)
    field_values.append(totalresult*100) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import cpu_count [as 别名]
def train(env_id, num_timesteps, seed, policy):

    ncpu = multiprocessing.cpu_count()
    if sys.platform == 'darwin': ncpu //= 2
    config = tf.ConfigProto(allow_soft_placement=True,
                            intra_op_parallelism_threads=ncpu,
                            inter_op_parallelism_threads=ncpu)
    config.gpu_options.allow_growth = True #pylint: disable=E1101
    tf.Session(config=config).__enter__()

    env = VecFrameStack(make_atari_env(env_id, 8, seed), 4)
    policy = {'cnn' : CnnPolicy, 'lstm' : LstmPolicy, 'lnlstm' : LnLstmPolicy}[policy]
    ppo2.learn(policy=policy, env=env, nsteps=128, nminibatches=4,
        lam=0.95, gamma=0.99, noptepochs=4, log_interval=1,
        ent_coef=.01,
        lr=lambda f : f * 2.5e-4,
        cliprange=lambda f : f * 0.1,
        total_timesteps=int(num_timesteps * 1.1)) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import cpu_count [as 别名]
def scrape_recipe_box(scraper, site_str, page_iter, status_interval=50):

    if args.append:
        recipes = quick_load(site_str)
    else:
        recipes = {}
    start = time.time()
    if args.multi:
        pool = Pool(cpu_count() * 2)
        results = pool.map(scraper, page_iter)
        for r in results:
            recipes.update(r)
    else:
        for i in page_iter:
            recipes.update(scraper(i))
            if i % status_interval == 0:
                print('Scraping page {} of {}'.format(i, max(page_iter)))
                quick_save(site_str, recipes)
            time.sleep(args.sleep)

    print('Scraped {} recipes from {} in {:.0f} minutes'.format(
        len(recipes), site_str, (time.time() - start) / 60))
    quick_save(site_str, recipes) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import cpu_count [as 别名]
def test_multiprocessing(app):
    """Tests that the number of children we produce is correct"""
    # Selects a number at random so we can spot check
    num_workers = random.choice(range(2, multiprocessing.cpu_count() * 2 + 1))
    process_list = set()

    def stop_on_alarm(*args):
        for process in multiprocessing.active_children():
            process_list.add(process.pid)
            process.terminate()

    signal.signal(signal.SIGALRM, stop_on_alarm)
    signal.alarm(3)
    app.run(HOST, PORT, workers=num_workers)

    assert len(process_list) == num_workers 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import cpu_count [as 别名]
def test_multiprocessing_with_blueprint(app):
    # Selects a number at random so we can spot check
    num_workers = random.choice(range(2, multiprocessing.cpu_count() * 2 + 1))
    process_list = set()

    def stop_on_alarm(*args):
        for process in multiprocessing.active_children():
            process_list.add(process.pid)
            process.terminate()

    signal.signal(signal.SIGALRM, stop_on_alarm)
    signal.alarm(3)

    bp = Blueprint("test_text")
    app.blueprint(bp)
    app.run(HOST, PORT, workers=num_workers)

    assert len(process_list) == num_workers


# this function must be outside a test function so that it can be
# able to be pickled (local functions cannot be pickled). 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import cpu_count [as 别名]
def load_config(config_data):
    config_data['pywren']['runtime'] = RUNTIME_NAME_DEFAULT
    config_data['pywren']['runtime_memory'] = None
    if 'runtime_timeout' not in config_data['pywren']:
        config_data['pywren']['runtime_timeout'] = RUNTIME_TIMEOUT_DEFAULT

    if 'storage_backend' not in config_data['pywren']:
        config_data['pywren']['storage_backend'] = 'localhost'

    if 'localhost' not in config_data:
        config_data['localhost'] = {}

    if 'ibm_cos' in config_data and 'private_endpoint' in config_data['ibm_cos']:
        del config_data['ibm_cos']['private_endpoint']

    if 'workers' in config_data['pywren']:
        config_data['localhost']['workers'] = config_data['pywren']['workers']
    else:
        total_cores = multiprocessing.cpu_count()
        config_data['pywren']['workers'] = total_cores
        config_data['localhost']['workers'] = total_cores 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import cpu_count [as 别名]
def get_params_for_mp(n_triples):
    n_cores = mp.cpu_count()
    pool = mp.Pool(n_cores)
    avg = n_triples // n_cores

    range_list = []
    start = 0
    for i in range(n_cores):
        num = avg + 1 if i < n_triples - avg * n_cores else avg
        range_list.append([start, start + num])
        start += num

    return n_cores, pool, range_list


# input: [(h1, {t1, t2 ...}), (h2, {t3 ...}), ...]
# output: {(h1, t1): paths, (h1, t2): paths, (h2, t3): paths, ...} 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import cpu_count [as 别名]
def cpu_count():
    """Return the number of CPU cores."""
    try:
        return multiprocessing.cpu_count()
    # TODO: remove except clause once we support only python >= 2.6
    except NameError:
        ## This code part is taken from parallel python.
        # Linux, Unix and MacOS
        if hasattr(os, "sysconf"):
            if "SC_NPROCESSORS_ONLN" in os.sysconf_names:
                # Linux & Unix
                n_cpus = os.sysconf("SC_NPROCESSORS_ONLN")
                if isinstance(n_cpus, int) and n_cpus > 0:
                    return n_cpus
            else:
                # OSX
                return int(os.popen2("sysctl -n hw.ncpu")[1].read())
        # Windows
        if "NUMBER_OF_PROCESSORS" in os.environ:
            n_cpus = int(os.environ["NUMBER_OF_PROCESSORS"])
            if n_cpus > 0:
                return n_cpus
        # Default
        return 1 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import cpu_count [as 别名]
def create_parser():
    parser = ArgumentParser(description=__doc__,
                            formatter_class=RawDescriptionHelpFormatter)
    parser.add_argument('--debug', action='store_true')
    parser.add_argument('--delimiter')
    parser.add_argument('--embedding-size', default=200, type=int)
    parser.add_argument('--graph-path')
    parser.add_argument('--has-header', action='store_true')
    parser.add_argument('--input', '-i', dest='infile', required=True)
    parser.add_argument('--log-level', '-l', type=str.upper, default='INFO')
    parser.add_argument('--num-walks', default=1, type=int)
    parser.add_argument('--model', '-m', dest='model_path')
    parser.add_argument('--output', '-o', dest='outfile', required=True)
    parser.add_argument('--stats', action='store_true')
    parser.add_argument('--undirected', action='store_true')
    parser.add_argument('--walk-length', default=10, type=int)
    parser.add_argument('--window-size', default=5, type=int)
    parser.add_argument('--workers', default=multiprocessing.cpu_count(),
                        type=int)
    return parser 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import cpu_count [as 别名]
def load_settings():
    with open('SETTINGS.json') as f:
        settings = json.load(f)

    data_dir = str(settings['competition-data-dir'])
    cache_dir = str(settings['data-cache-dir'])
    submission_dir = str(settings['submission-dir'])
    N_jobs = str(settings['num-jobs'])
    N_jobs = multiprocessing.cpu_count() if N_jobs == 'auto' else int(N_jobs)

    for d in (cache_dir, submission_dir):
        try:
            os.makedirs(d)
        except:
            pass

    return Settings(data_dir=data_dir, cache_dir=cache_dir, submission_dir=submission_dir, N_jobs=N_jobs) 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import cpu_count [as 别名]
def train_reader(train_list_path):

    def reader():
        with open(train_list_path, 'r') as f:
            lines = f.readlines()
            # 打乱数据
            np.random.shuffle(lines)
            # 开始获取每张图像和标签
            for line in lines:
                data, label = line.split('\t')
                yield data, label

    return paddle.reader.xmap_readers(train_mapper, reader, cpu_count(), 1024)


# 测试数据的预处理 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import cpu_count [as 别名]
def train_reader(train_list_path, crop_size, resize_size):
    father_path = os.path.dirname(train_list_path)

    def reader():
        with open(train_list_path, 'r') as f:
            lines = f.readlines()
            # 打乱图像列表
            np.random.shuffle(lines)
            # 开始获取每张图像和标签
            for line in lines:
                img, label = line.split('\t')
                img = os.path.join(father_path, img)
                yield img, label, crop_size, resize_size

    return paddle.reader.xmap_readers(train_mapper, reader, cpu_count(), 102400)


# 测试图片的预处理 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import cpu_count [as 别名]
def cpu_count_physical():
    """
    tries to get the number of physical (ie not virtual) cores
    """
    try:
        import psutil
        return psutil.cpu_count(logical=False)
    except:
        import multiprocessing
        return multiprocessing.cpu_count() 

# 需要导入模块: import multiprocessing [as 别名]
# 或者: from multiprocessing import cpu_count [as 别名]
def _n_workers_for_local_cluster(calcs):
    """The number of workers used in a LocalCluster

    An upper bound is set at the cpu_count or the number of calcs submitted,
    depending on which is smaller.  This is to prevent more workers from
    being started than needed (but also to prevent too many workers from
    being started in the case that a large number of calcs are submitted).
    """
    return min(cpu_count(), len(calcs))