本文整理汇总了Python中data.DataLoader方法的典型用法代码示例。如果您正苦于以下问题：Python data.DataLoader方法的具体用法？Python data.DataLoader怎么用？Python data.DataLoader使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类data的用法示例。
在下文中一共展示了data.DataLoader方法的2个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: run_test
# 需要导入模块: import data [as 别名]
# 或者: from data import DataLoader [as 别名]
def run_test(epoch=-1):
print('Running Test')
opt = TestOptions().parse()
opt.serial_batches = True # no shuffle
dataset = DataLoader(opt)
model = create_model(opt)
writer = Writer(opt)
# test
writer.reset_counter()
for i, data in enumerate(dataset):
model.set_input(data)
ncorrect, nexamples = model.test()
writer.update_counter(ncorrect, nexamples)
writer.print_acc(epoch, writer.acc)
return writer.acc 示例2: run
# 需要导入模块: import data [as 别名]
# 或者: from data import DataLoader [as 别名]
def run(args, local_rank):
""" Distributed Synchronous """
torch.manual_seed(1234)
vocab = Vocab(args.vocab, min_occur_cnt=args.min_occur_cnt, specials=[])
if (args.world_size == 1 or dist.get_rank() == 0):
print ("vocab.size = %d"%vocab.size, flush=True)
model = BIGLM(local_rank, vocab, args.embed_dim, args.ff_embed_dim,\
args.num_heads, args.dropout, args.layers, args.smoothing, args.approx)
if args.start_from is not None:
ckpt = torch.load(args.start_from, map_location='cpu')
model.load_state_dict(ckpt['model'])
model = model.cuda(local_rank)
if args.world_size > 1:
torch.manual_seed(1234 + dist.get_rank())
random.seed(5678 + dist.get_rank())
optimizer = Optim(model.embed_dim, args.lr, args.warmup_steps, torch.optim.Adam(model.parameters(), lr=0, betas=(0.9, 0.998), eps=1e-9))
if args.start_from is not None:
optimizer.load_state_dict(ckpt['optimizer'])
#train_data = DataLoader(vocab, args.train_data+"0"+str(local_rank), args.batch_size, args.max_len, args.min_len)
train_data = DataLoader(vocab, args.train_data, args.batch_size, args.max_len, args.min_len)
batch_acm = 0
acc_acm, nll_acm, ppl_acm, ntokens_acm, nxs, npairs_acm, loss_acm = 0., 0., 0., 0., 0., 0., 0.
while True:
model.train()
for truth, inp, msk in train_data:
batch_acm += 1
truth = truth.cuda(local_rank)
inp = inp.cuda(local_rank)
msk = msk.cuda(local_rank)
model.zero_grad()
res, loss, acc, nll, ppl, ntokens, npairs = model(truth, inp, msk)
loss_acm += loss.item()
acc_acm += acc
nll_acm += nll
ppl_acm += ppl
ntokens_acm += ntokens
npairs_acm += npairs
nxs += npairs
loss.backward()
if args.world_size > 1:
average_gradients(model)
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
if (args.world_size==1 or dist.get_rank() ==0) and batch_acm%args.print_every == -1%args.print_every:
print ('batch_acm %d, loss %.3f, acc %.3f, nll %.3f, ppl %.3f, x_acm %d, lr %.6f'\
%(batch_acm, loss_acm/args.print_every, acc_acm/ntokens_acm, \
nll_acm/nxs, ppl_acm/nxs, npairs_acm, optimizer._rate), flush=True)
acc_acm, nll_acm, ppl_acm, ntokens_acm, loss_acm, nxs = 0., 0., 0., 0., 0., 0.
if (args.world_size==1 or dist.get_rank() ==0) and batch_acm%args.save_every == -1%args.save_every:
if not os.path.exists(args.save_dir):
os.mkdir(args.save_dir)
torch.save({'args':args, 'model':model.state_dict(), 'optimizer':optimizer.state_dict()}, '%s/epoch%d_batch_%d'%(args.save_dir, train_data.epoch_id, batch_acm))