本文整理匯總了Python中transformers.AdamW方法的典型用法代碼示例。如果您正苦於以下問題：Python transformers.AdamW方法的具體用法？Python transformers.AdamW怎麽用？Python transformers.AdamW使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類transformers的用法示例。
在下文中一共展示了transformers.AdamW方法的5個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: get_default_optimizer
# 需要導入模塊: import transformers [as 別名]
# 或者: from transformers import AdamW [as 別名]
def get_default_optimizer(model, weight_decay, learning_rate, adam_epsilon):
no_decay = ["bias", "LayerNorm.weight"]
optimizer_grouped_parameters = [
{
"params": [
p
for n, p in model.named_parameters()
if not any(nd in n for nd in no_decay)
],
"weight_decay": weight_decay,
},
{
"params": [
p
for n, p in model.named_parameters()
if any(nd in n for nd in no_decay)
],
"weight_decay": 0.0,
},
]
optimizer = AdamW(
optimizer_grouped_parameters, lr=learning_rate, eps=adam_epsilon
)
return optimizer 示例2: create_optimizer
# 需要導入模塊: import transformers [as 別名]
# 或者: from transformers import AdamW [as 別名]
def create_optimizer(args, parameters):
"""
Creates an adam optimizer.
"""
optimizer = AdamW(
lr=args.lr,
params=parameters,
weight_decay=0.01)
return optimizer
# implementation is from DialoGPT repo 示例3: get_optimizer
# 需要導入模塊: import transformers [as 別名]
# 或者: from transformers import AdamW [as 別名]
def get_optimizer(model, lr, weight_decay, model_type='siamese'):
param_groups = get_optimizer_param_groups(model.head, lr, weight_decay)
if model_type == 'siamese':
param_groups += get_optimizer_param_groups(model.transformer, lr / 100, weight_decay)
elif model_type == 'double':
param_groups += get_optimizer_param_groups(model.q_transformer, lr / 100, weight_decay)
param_groups += get_optimizer_param_groups(model.a_transformer, lr / 100, weight_decay)
return AdamW(param_groups) 示例4: test_adam_w
# 需要導入模塊: import transformers [as 別名]
# 或者: from transformers import AdamW [as 別名]
def test_adam_w(self):
w = torch.tensor([0.1, -0.2, -0.1], requires_grad=True)
target = torch.tensor([0.4, 0.2, -0.5])
criterion = torch.nn.MSELoss()
# No warmup, constant schedule, no gradient clipping
optimizer = AdamW(params=[w], lr=2e-1, weight_decay=0.0)
for _ in range(100):
loss = criterion(w, target)
loss.backward()
optimizer.step()
w.grad.detach_() # No zero_grad() function on simple tensors. we do it ourselves.
w.grad.zero_()
self.assertListAlmostEqual(w.tolist(), [0.4, 0.2, -0.5], tol=1e-2) 示例5: prepare_for_training
# 需要導入模塊: import transformers [as 別名]
# 或者: from transformers import AdamW [as 別名]
def prepare_for_training(args, model, checkpoint_state_dict, amp):
no_decay = ['bias', 'LayerNorm.weight']
optimizer_grouped_parameters = [
{'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],
'weight_decay': args.weight_decay},
{'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
]
optimizer = AdamW(optimizer_grouped_parameters, lr=args.learning_rate, eps=args.adam_epsilon)
if amp:
model, optimizer = amp.initialize(model, optimizer, opt_level=args.fp16_opt_level)
if checkpoint_state_dict:
amp.load_state_dict(checkpoint_state_dict['amp'])
if checkpoint_state_dict:
optimizer.load_state_dict(checkpoint_state_dict['optimizer'])
model.load_state_dict(checkpoint_state_dict['model'])
# multi-gpu training (should be after apex fp16 initialization)
if args.n_gpu > 1:
model = torch.nn.DataParallel(model)
# Distributed training (should be after apex fp16 initialization)
if args.local_rank != -1:
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True)
return model, optimizer