本文整理汇总了Python中torch.multiprocessing.cpu_count方法的典型用法代码示例。如果您正苦于以下问题:Python multiprocessing.cpu_count方法的具体用法?Python multiprocessing.cpu_count怎么用?Python multiprocessing.cpu_count使用的例子?那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类torch.multiprocessing
的用法示例。
在下文中一共展示了multiprocessing.cpu_count方法的6个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: calc_chunksize
# 需要导入模块: from torch import multiprocessing [as 别名]
# 或者: from torch.multiprocessing import cpu_count [as 别名]
def calc_chunksize(num_dicts, min_chunksize=4, max_chunksize=2000, max_processes=128):
num_cpus = min(mp.cpu_count() - 1 or 1, max_processes) # -1 to keep a CPU core free for the main process
dicts_per_cpu = np.ceil(num_dicts / num_cpus)
# automatic adjustment of multiprocessing chunksize
# for small files (containing few dicts) we want small chunksize to ulitize all available cores but never less
# than 2, because we need it to sample another random sentence in LM finetuning
# for large files we want to minimize processor spawning without giving too much data to one process, so we
# clip it at 5k
multiprocessing_chunk_size = int(np.clip((np.ceil(dicts_per_cpu / 5)), a_min=min_chunksize, a_max=max_chunksize))
# This lets us avoid cases in lm_finetuning where a chunk only has a single doc and hence cannot pick
# a valid next sentence substitute from another document
if num_dicts != 1:
while num_dicts % multiprocessing_chunk_size == 1:
multiprocessing_chunk_size -= -1
dict_batches_to_process = int(num_dicts / multiprocessing_chunk_size)
num_processes = min(num_cpus, dict_batches_to_process) or 1
return multiprocessing_chunk_size, num_processes
示例2: main
# 需要导入模块: from torch import multiprocessing [as 别名]
# 或者: from torch.multiprocessing import cpu_count [as 别名]
def main():
env = gym.make(env_name)
env.seed(500)
torch.manual_seed(500)
num_inputs = env.observation_space.shape[0]
num_actions = env.action_space.n
print('state size:', num_inputs)
print('action size:', num_actions)
online_net = QNet(num_inputs, num_actions)
target_net = QNet(num_inputs, num_actions)
target_net.load_state_dict(online_net.state_dict())
online_net.share_memory()
target_net.share_memory()
optimizer = SharedAdam(online_net.parameters(), lr=lr)
global_ep, global_ep_r, res_queue = mp.Value('i', 0), mp.Value('d', 0.), mp.Queue()
writer = SummaryWriter('logs')
online_net.to(device)
target_net.to(device)
online_net.train()
target_net.train()
workers = [Worker(online_net, target_net, optimizer, global_ep, global_ep_r, res_queue, i) for i in range(mp.cpu_count())]
[w.start() for w in workers]
res = []
while True:
r = res_queue.get()
if r is not None:
res.append(r)
[ep, ep_r, loss] = r
writer.add_scalar('log/score', float(ep_r), ep)
writer.add_scalar('log/loss', float(loss), ep)
else:
break
[w.join() for w in workers]
示例3: main
# 需要导入模块: from torch import multiprocessing [as 别名]
# 或者: from torch.multiprocessing import cpu_count [as 别名]
def main():
env = gym.make(env_name)
env.seed(500)
torch.manual_seed(500)
num_inputs = env.observation_space.shape[0]
num_actions = env.action_space.n
env.close()
global_model = Model(num_inputs, num_actions)
global_average_model = Model(num_inputs, num_actions)
global_model.share_memory()
global_average_model.share_memory()
global_optimizer = SharedAdam(global_model.parameters(), lr=lr)
global_ep, global_ep_r, res_queue = mp.Value('i', 0), mp.Value('d', 0.), mp.Queue()
writer = SummaryWriter('logs')
n = mp.cpu_count()
workers = [Worker(global_model, global_average_model, global_optimizer, global_ep, global_ep_r, res_queue, i) for i in range(n)]
[w.start() for w in workers]
res = []
while True:
r = res_queue.get()
if r is not None:
res.append(r)
[ep, ep_r, loss] = r
writer.add_scalar('log/score', float(ep_r), ep)
writer.add_scalar('log/loss', float(loss), ep)
else:
break
[w.join() for w in workers]
示例4: main
# 需要导入模块: from torch import multiprocessing [as 别名]
# 或者: from torch.multiprocessing import cpu_count [as 别名]
def main():
env = gym.make(env_name)
env.seed(500)
torch.manual_seed(500)
num_inputs = env.observation_space.shape[0]
num_actions = env.action_space.n
global_model = Model(num_inputs, num_actions)
global_model.share_memory()
global_optimizer = SharedAdam(global_model.parameters(), lr=lr)
global_ep, global_ep_r, res_queue = mp.Value('i', 0), mp.Value('d', 0.), mp.Queue()
writer = SummaryWriter('logs')
workers = [Worker(global_model, global_optimizer, global_ep, global_ep_r, res_queue, i) for i in range(mp.cpu_count())]
[w.start() for w in workers]
res = []
while True:
r = res_queue.get()
if r is not None:
res.append(r)
[ep, ep_r, loss] = r
writer.add_scalar('log/score', float(ep_r), ep)
writer.add_scalar('log/loss', float(loss), ep)
else:
break
[w.join() for w in workers]
示例5: run_MCTS
# 需要导入模块: from torch import multiprocessing [as 别名]
# 或者: from torch.multiprocessing import cpu_count [as 别名]
def run_MCTS(args, start_idx=0, iteration=0):
net_to_play="%s_iter%d.pth.tar" % (args.neural_net_name, iteration)
net = ConnectNet()
cuda = torch.cuda.is_available()
if cuda:
net.cuda()
if args.MCTS_num_processes > 1:
logger.info("Preparing model for multi-process MCTS...")
mp.set_start_method("spawn",force=True)
net.share_memory()
net.eval()
current_net_filename = os.path.join("./model_data/",\
net_to_play)
if os.path.isfile(current_net_filename):
checkpoint = torch.load(current_net_filename)
net.load_state_dict(checkpoint['state_dict'])
logger.info("Loaded %s model." % current_net_filename)
else:
torch.save({'state_dict': net.state_dict()}, os.path.join("./model_data/",\
net_to_play))
logger.info("Initialized model.")
processes = []
if args.MCTS_num_processes > mp.cpu_count():
num_processes = mp.cpu_count()
logger.info("Required number of processes exceed number of CPUs! Setting MCTS_num_processes to %d" % num_processes)
else:
num_processes = args.MCTS_num_processes
logger.info("Spawning %d processes..." % num_processes)
with torch.no_grad():
for i in range(num_processes):
p = mp.Process(target=MCTS_self_play, args=(net, args.num_games_per_MCTS_process, start_idx, i, args, iteration))
p.start()
processes.append(p)
for p in processes:
p.join()
logger.info("Finished multi-process MCTS!")
elif args.MCTS_num_processes == 1:
logger.info("Preparing model for MCTS...")
net.eval()
current_net_filename = os.path.join("./model_data/",\
net_to_play)
if os.path.isfile(current_net_filename):
checkpoint = torch.load(current_net_filename)
net.load_state_dict(checkpoint['state_dict'])
logger.info("Loaded %s model." % current_net_filename)
else:
torch.save({'state_dict': net.state_dict()}, os.path.join("./model_data/",\
net_to_play))
logger.info("Initialized model.")
with torch.no_grad():
MCTS_self_play(net, args.num_games_per_MCTS_process, start_idx, 0, args, iteration)
logger.info("Finished MCTS!")
示例6: propagate
# 需要导入模块: from torch import multiprocessing [as 别名]
# 或者: from torch.multiprocessing import cpu_count [as 别名]
def propagate(nnf, feat_A, feat_AP, feat_B, feat_BP, patch_size, iters=2, rand_search_radius=200):
print("\tpatch_size:{}; num_iters:{}; rand_search_radius:{}".format(patch_size, iters, rand_search_radius))
nnd = np.zeros(nnf.shape[:2])
A_size = feat_A.shape[:2]
B_size = feat_B.shape[:2]
for ay in range(A_size[0]):
for ax in range(A_size[1]):
by, bx = nnf[ay, ax]
nnd[ay, ax] = cal_dist(ay, ax, by, bx, feat_A, feat_AP, feat_B, feat_BP, A_size, B_size, patch_size)
manager = mp.Manager()
q = manager.Queue(A_size[1] * A_size[0])
cpus = min(mp.cpu_count(), A_size[0] // 20 + 1)
for i in range(iters):
p = Pool(cpus)
ay_start = 0
while ay_start < A_size[0]:
ax_start = 0
while ax_start < A_size[1]:
p.apply_async(pixelmatch, args=(q, ax_start, ay_start,
cpus,
nnf, nnd,
A_size, B_size,
feat_A, feat_AP,
feat_B, feat_BP,
patch_size,
rand_search_radius,))
ax_start += A_size[1] // cpus + 1
ay_start += A_size[0] // cpus + 1
p.close()
p.join()
while not q.empty():
ax, ay, xbest, ybest, dbest = q.get()
nnf[ay, ax] = np.array([ybest, xbest])
nnd[ay, ax] = dbest
return nnf, nnd