本文整理匯總了Python中torch.nn.AdaptiveLogSoftmaxWithLoss方法的典型用法代碼示例。如果您正苦於以下問題：Python nn.AdaptiveLogSoftmaxWithLoss方法的具體用法？Python nn.AdaptiveLogSoftmaxWithLoss怎麽用？Python nn.AdaptiveLogSoftmaxWithLoss使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類torch.nn的用法示例。
在下文中一共展示了nn.AdaptiveLogSoftmaxWithLoss方法的5個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: __init__
# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import AdaptiveLogSoftmaxWithLoss [as 別名]
def __init__(self, config):
super(XLMPredLayer, self).__init__()
self.asm = config.asm
self.n_words = config.n_words
self.pad_index = config.pad_index
dim = config.emb_dim
if config.asm is False:
self.proj = nn.Linear(dim, config.n_words, bias=True)
else:
self.proj = nn.AdaptiveLogSoftmaxWithLoss(
in_features=dim,
n_classes=config.n_words,
cutoffs=config.asm_cutoffs,
div_value=config.asm_div_value,
head_bias=True, # default is False
) 示例2: __init__
# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import AdaptiveLogSoftmaxWithLoss [as 別名]
def __init__(self, config):
super().__init__()
self.asm = config.asm
self.n_words = config.n_words
self.pad_index = config.pad_index
dim = config.emb_dim
if config.asm is False:
self.proj = nn.Linear(dim, config.n_words, bias=True)
else:
self.proj = nn.AdaptiveLogSoftmaxWithLoss(
in_features=dim,
n_classes=config.n_words,
cutoffs=config.asm_cutoffs,
div_value=config.asm_div_value,
head_bias=True, # default is False
) 示例3: __init__
# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import AdaptiveLogSoftmaxWithLoss [as 別名]
def __init__(self, params):
super().__init__()
self.asm = params.asm
self.n_words = params.n_words
self.pad_index = params.pad_index
self.label_smoothing = params.label_smoothing if hasattr(params, "label_smoothing") else 0.0
dim = params.emb_dim
if params.asm is False:
self.proj = Linear(dim, params.n_words, bias=True)
else:
self.proj = nn.AdaptiveLogSoftmaxWithLoss(
in_features=dim,
n_classes=params.n_words,
cutoffs=params.asm_cutoffs,
div_value=params.asm_div_value,
head_bias=True, # default is False
) 示例4: __init__
# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import AdaptiveLogSoftmaxWithLoss [as 別名]
def __init__(self, local_rank, vocab, embed_dim, ff_embed_dim, num_heads, dropout, layers, smoothing_factor, approx):
super(BIGLM, self).__init__()
self.vocab = vocab
self.embed_dim = embed_dim
self.tok_embed = Embedding(self.vocab.size, embed_dim, self.vocab.padding_idx)
self.pos_embed = LearnedPositionalEmbedding(embed_dim, device=local_rank)
self.layers = nn.ModuleList()
for i in range(layers):
self.layers.append(TransformerLayer(embed_dim, ff_embed_dim, num_heads, dropout))
self.emb_layer_norm = LayerNorm(embed_dim)
self.one_more = nn.Linear(embed_dim, embed_dim)
self.one_more_layer_norm = LayerNorm(embed_dim)
self.out_proj = nn.Linear(embed_dim, self.vocab.size)
self.attn_mask = SelfAttentionMask(device=local_rank)
self.smoothing = LabelSmoothing(local_rank, self.vocab.size, self.vocab.padding_idx, smoothing_factor)
self.dropout = dropout
self.device = local_rank
if approx == "none":
self.approx = None
elif approx == "adaptive":
self.approx = nn.AdaptiveLogSoftmaxWithLoss(self.embed_dim, self.vocab.size, [10000, 20000, 200000])
else:
raise NotImplementedError("%s has not been implemented"%approx)
self.reset_parameters() 示例5: __init__
# 需要導入模塊: from torch import nn [as 別名]
# 或者: from torch.nn import AdaptiveLogSoftmaxWithLoss [as 別名]
def __init__(self, args, save_path=None):
super(LMBase, self).__init__()
logger.info(self.__class__.__name__)
self.lm_type = args.lm_type
self.save_path = save_path
self.d_model = args.transformer_d_model
self.n_layers = args.n_layers
self.n_heads = args.transformer_n_heads
self.lsm_prob = args.lsm_prob
self.tie_embedding = args.tie_embedding
self.mem_len = args.mem_len
if args.recog_mem_len > 0:
self.mem_len = args.recog_mem_len
self.vocab = args.vocab
self.eos = 2
self.pad = 3
# NOTE: reserved in advance
# for cache
self.cache_theta = 0.2 # smoothing parameter
self.cache_lambda = 0.2 # cache weight
self.cache_ids = []
self.cache_keys = []
self.cache_attn = []
self.embed = nn.Embedding(self.vocab, self.d_model, padding_idx=self.pad)
self.pos_enc = PositionalEncoding(self.d_model, args.dropout_in, args.transformer_pe_type,
args.transformer_param_init)
self.layers = nn.ModuleList([copy.deepcopy(TransformerDecoderBlock(
self.d_model, args.transformer_d_ff, 'scaled_dot',
self.n_heads, args.dropout_hidden, args.dropout_att, args.dropout_layer,
args.transformer_layer_norm_eps, args.transformer_ffn_activation, args.transformer_param_init,
src_tgt_attention=False)) for lth in range(self.n_layers)])
self.norm_out = nn.LayerNorm(self.d_model, eps=args.transformer_layer_norm_eps)
self.adaptive_softmax = None
self.output = None
if args.adaptive_softmax:
self.adaptive_softmax = nn.AdaptiveLogSoftmaxWithLoss(
self.d_model, self.vocab,
cutoffs=[round(self.vocab / 15), 3 * round(self.vocab / 15)],
# cutoffs=[self.vocab // 25, 3 * self.vocab // 5],
div_value=4.0)
else:
self.output = nn.Linear(self.d_model, self.vocab)
if args.tie_embedding:
self.output.weight = self.embed.weight
self.reset_parameters()