本文整理汇总了Python中torchtext.vocab.GloVe方法的典型用法代码示例。如果您正苦于以下问题:Python vocab.GloVe方法的具体用法?Python vocab.GloVe怎么用?Python vocab.GloVe使用的例子?那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类torchtext.vocab
的用法示例。
在下文中一共展示了vocab.GloVe方法的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: __init__
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import GloVe [as 别名]
def __init__(self, emb_dim=50, mbsize=32):
self.TEXT = data.Field(init_token='<start>', eos_token='<eos>', lower=True, tokenize='spacy', fix_length=16)
self.LABEL = data.Field(sequential=False, unk_token=None)
# Only take sentences with length <= 15
f = lambda ex: len(ex.text) <= 15 and ex.label != 'neutral'
train, val, test = datasets.SST.splits(
self.TEXT, self.LABEL, fine_grained=False, train_subtrees=False,
filter_pred=f
)
self.TEXT.build_vocab(train, vectors=GloVe('6B', dim=emb_dim))
self.LABEL.build_vocab(train)
self.n_vocab = len(self.TEXT.vocab.itos)
self.emb_dim = emb_dim
self.train_iter, self.val_iter, _ = data.BucketIterator.splits(
(train, val, test), batch_size=mbsize, device=-1,
shuffle=True, repeat=True
)
self.train_iter = iter(self.train_iter)
self.val_iter = iter(self.val_iter)
示例2: test_vocab_download_glove_vectors
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import GloVe [as 别名]
def test_vocab_download_glove_vectors(self):
c = Counter({'hello': 4, 'world': 3, 'ᑌᑎIᑕOᗪᕮ_Tᕮ᙭T': 5, 'freq_too_low': 2})
# Build a vocab and get vectors twice to test caching, then once more
# to test string aliases.
for i in range(3):
if i == 2:
vectors = "glove.twitter.27B.25d"
else:
vectors = GloVe(name='twitter.27B', dim='25')
v = vocab.Vocab(c, min_freq=3, specials=['<unk>', '<pad>', '<bos>'],
vectors=vectors)
expected_itos = ['<unk>', '<pad>', '<bos>',
'ᑌᑎIᑕOᗪᕮ_Tᕮ᙭T', 'hello', 'world']
expected_stoi = {x: index for index, x in enumerate(expected_itos)}
self.assertEqual(v.itos, expected_itos)
self.assertEqual(dict(v.stoi), expected_stoi)
vectors = v.vectors.numpy()
# The first 5 entries in each vector.
expected_twitter = {
'hello': [-0.77069, 0.12827, 0.33137, 0.0050893, -0.47605],
'world': [0.10301, 0.095666, -0.14789, -0.22383, -0.14775],
}
for word in expected_twitter:
assert_allclose(vectors[v.stoi[word], :5],
expected_twitter[word])
assert_allclose(vectors[v.stoi['<unk>']], np.zeros(25))
assert_allclose(vectors[v.stoi['OOV token']], np.zeros(25))
# Delete the vectors after we're done to save disk space on CI
if os.environ.get("TRAVIS") == "true":
zip_file = os.path.join(self.project_root, ".vector_cache",
"glove.twitter.27B.zip")
conditional_remove(zip_file)
for dim in ["25", "50", "100", "200"]:
conditional_remove(os.path.join(self.project_root, ".vector_cache",
"glove.twitter.27B.{}d.txt".format(dim)))
示例3: load_data
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import GloVe [as 别名]
def load_data(batch_size=32):
# define a tokenizer
# tokenize = lambda s : nltk.word_tokenize(s)
tokenize = lambda s : s.split()
# fields : ( text_field, label_field )
print(':: creating fields')
text_field = data.Field(sequential=True, tokenize=tokenize, lower=True, include_lengths=True, batch_first=True, fix_length=200)
#text_field = data.Field(sequential=True, tokenize=tokenize, lower=True)
label_field = data.LabelField(sequential=False)
# get IMDB data
print(':: fetching IMDB data')
train_data, test_data = datasets.IMDB.splits(text_field, label_field)
# build vocabulary for fields
text_field.build_vocab(train_data, vectors=GloVe(name='6B', dim=300))
label_field.build_vocab(train_data)
# split train into train and valid
train_data, valid_data = train_data.split()
print(':: labels :', label_field.vocab.stoi)
# iterators
train_iter, test_iter, valid_iter = data.BucketIterator.splits(
(train_data, test_data, valid_data),
batch_size=batch_size,
sort_key=lambda x : len(x.text),
repeat=False,
shuffle=True)
return ( (text_field, label_field), (train_iter, test_iter, valid_iter),
text_field.vocab.vectors, # GloVe vectors
len(text_field.vocab)
)
示例4: __init__
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import GloVe [as 别名]
def __init__(self, word2prob, arora_a, glove_name, glove_dim, first_sv, data_path):
"""
Inputs:
word2prob: dict mapping words to their unigram probs
arora_a: a float. Is the constant (called "a" in the paper)
used to compute Arora sentence embeddings.
glove_name: the version of GloVe to use, e.g. '840B'
glove_dim: the dimension of the GloVe embeddings to use, e.g. 300
first_sv: np array shape (glove_dim). The first singular value,
used to compute Arora sentence embeddings. Can be None.
data_path: The data path (we will use this to download glove)
"""
self.word2prob = word2prob
self.arora_a = arora_a
self.glove_name = glove_name
self.glove_dim = glove_dim
self.first_sv = first_sv
self.data_path = data_path
if self.first_sv is not None:
self.first_sv = torch.tensor(self.first_sv) # convert to torch tensor
self.min_word_prob = min(word2prob.values()) # prob of rarest word
self.tt_embs = None # will be torchtext.vocab.GloVe object
self.emb_matrix = None # will be np array shape (vocab_size, glove_dim)
# Initialize a cache, which holds up to 64 sentences, along with their
# corresponding word similarity scores (i.e. cosine sim for every word in the
# vocab). This enables us to repeatedly retrieve sims for sentences we have
# already processed (useful for batched beam search).
self.cache_limit = 64
self.cache_sent2sims = {} # maps sent to sims. holds up to cache_limit.
self.cache_sentqueue = deque() # list of sents. add to right, remove from left
示例5: get_glove_embs
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import GloVe [as 别名]
def get_glove_embs(self):
"""
Loads torchtext GloVe embs from file and stores in self.tt_embs.
"""
if not hasattr(self, 'glove_cache'):
self.glove_cache = modelzoo_path(self.data_path, 'models:glove_vectors')
print('Loading torchtext GloVe embs (for Arora sentence embs)...')
self.tt_embs = vocab.GloVe(
name=self.glove_name, dim=self.glove_dim, cache=self.glove_cache
)
print('Finished loading torchtext GloVe embs')
示例6: get_emb_matrix
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import GloVe [as 别名]
def get_emb_matrix(self, dictionary):
"""
Construct an embedding matrix containing pretrained GloVe vectors for all words
in dictionary, and store in self.emb_matrix. This is needed for response-
relatedness weighted decoding.
Inputs:
dictionary: ParlAI dictionary
"""
print(
'Constructing GloVe emb matrix for response-relatedness weighted '
'decoding...'
)
self.emb_matrix = []
oov_indices = [] # list of dictionary indices for all OOV words
for idx in range(len(dictionary)):
word = dictionary[idx]
if word in self.tt_embs.stoi:
word_emb = self.tt_embs.vectors[self.tt_embs.stoi[word]]
else:
# If word is OOV, enter a zero vector instead.
# This means that the cosine similarity will always be zero.
word_emb = torch.zeros(self.glove_dim)
oov_indices.append(idx)
self.emb_matrix.append(word_emb)
self.emb_matrix = np.stack(self.emb_matrix) # (vocab_size, glove_dim)
print(
'Done constructing GloVe emb matrix; found %i OOVs of %i words'
% (len(oov_indices), len(dictionary))
)
# Get the norm of each of the word vectors. This is needed for cosine sims.
# self.emb_matrix_norm is a np array shape (vocab_size)
self.emb_matrix_norm = np.linalg.norm(self.emb_matrix, axis=1)
# For the OOV words which have zero vectors,
# set the norm to 1.0 so we don't have divide-by-zero errors
for idx in oov_indices:
self.emb_matrix_norm[idx] = 1.0
示例7: get_word_sims
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import GloVe [as 别名]
def get_word_sims(self, sent, sent_emb, dictionary):
"""
Given a sentence and its Arora-style sentence embedding, compute the cosine
similarities to it, for all words in the dictionary.
Inputs:
sent: string. Used only for caching lookup purposes.
sent_emb: torch Tensor shape (glove_dim).
dictionary: ParlAI dictionary
Returns:
sims: torch Tensor shape (vocab_size), containing the cosine sims.
"""
# If we haven't initialized the GloVe emb matrix yet, do so
if self.emb_matrix is None:
self.get_emb_matrix(dictionary)
# If we have already computed sims for this sentence, return it
if sent in self.cache_sent2sims:
sims = self.cache_sent2sims[sent]
return sims
# Compute the cosine similarities. Implementation from here:
# https://codereview.stackexchange.com/questions/55717/efficient-numpy-cosine-distance-calculation
dotted = self.emb_matrix.dot(sent_emb) # shape (vocab_size)
sent_emb_norm = np.linalg.norm(sent_emb) # norm of the sent emb. scalar
norms = np.multiply(self.emb_matrix_norm, sent_emb_norm) # shape (vocab_size)
sims = np.divide(dotted, norms) # divide dot prods by norms. shape (vocab_size)
sims = torch.tensor(sims) # convert to torch Tensor, shape (vocab_size)
# Cache sims in self.cache_sent2sims
self.cache_sentqueue.append(sent) # append sent to right
self.cache_sent2sims[sent] = sims # add (sent, sims) pair to cache
if len(self.cache_sentqueue) > self.cache_limit:
to_remove = self.cache_sentqueue.popleft() # remove from left
del self.cache_sent2sims[to_remove] # remove from cache
assert len(self.cache_sent2sims) == len(self.cache_sentqueue)
assert len(self.cache_sent2sims) <= self.cache_limit
return sims
示例8: __init__
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import GloVe [as 别名]
def __init__(
self, word2prob, arora_a, glove_name, glove_dim, first_sv, glove_cache
):
"""
Inputs:
word2prob: dict mapping words to their unigram probs
arora_a: a float. Is the constant (called "a" in the paper)
used to compute Arora sentence embeddings.
glove_name: the version of GloVe to use, e.g. '840B'
glove_dim: the dimension of the GloVe embeddings to use, e.g. 300
first_sv: np array shape (glove_dim). The first singular value,
used to compute Arora sentence embeddings. Can be None.
glove_cache: The path to where the glove vectors are stored.
"""
self.word2prob = word2prob
self.arora_a = arora_a
self.glove_name = glove_name
self.glove_dim = glove_dim
self.glove_cache = glove_cache
self.first_sv = first_sv
if self.first_sv is not None:
self.first_sv = torch.tensor(self.first_sv) # convert to torch tensor
self.min_word_prob = min(word2prob.values()) # prob of rarest word
self.tt_embs = None # will be torchtext.vocab.GloVe object
self.emb_matrix = None # will be np array shape (vocab_size, glove_dim)
# Initialize a cache, which holds up to 64 sentences, along with their
# corresponding word similarity scores (i.e. cosine sim for every word in the
# vocab). This enables us to repeatedly retrieve sims for sentences we have
# already processed (useful for batched beam search).
self.cache_limit = 64
self.cache_sent2sims = {} # maps sent to sims. holds up to cache_limit.
self.cache_sentqueue = deque() # list of sents. add to right, remove from left
示例9: get_glove_embs
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import GloVe [as 别名]
def get_glove_embs(self):
"""
Loads torchtext GloVe embs from file and stores in self.tt_embs.
"""
print('Loading torchtext GloVe embs (for Arora sentence embs)...')
self.tt_embs = vocab.GloVe(
name=self.glove_name, dim=self.glove_dim, cache=self.glove_cache
)
print('Finished loading torchtext GloVe embs')
示例10: get_emb_matrix
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import GloVe [as 别名]
def get_emb_matrix(self, dictionary):
"""
Construct an embedding matrix containing pretrained GloVe vectors for all words
in dictionary, and store in self.emb_matrix. This is needed for
response-relatedness weighted decoding.
Inputs:
dictionary: ParlAI dictionary
"""
print(
'Constructing GloVe emb matrix for response-relatedness weighted '
'decoding...'
)
self.emb_matrix = []
oov_indices = [] # list of dictionary indices for all OOV words
for idx in range(len(dictionary)):
word = dictionary[idx]
if word in self.tt_embs.stoi:
word_emb = self.tt_embs.vectors[self.tt_embs.stoi[word]]
else:
# If word is OOV, enter a zero vector instead.
# This means that the cosine similarity will always be zero.
word_emb = torch.zeros(self.glove_dim)
oov_indices.append(idx)
self.emb_matrix.append(word_emb)
self.emb_matrix = np.stack(self.emb_matrix) # (vocab_size, glove_dim)
print(
'Done constructing GloVe emb matrix; found %i OOVs of %i words'
% (len(oov_indices), len(dictionary))
)
# Get the norm of each of the word vectors. This is needed for cosine sims.
# self.emb_matrix_norm is a np array shape (vocab_size)
self.emb_matrix_norm = np.linalg.norm(self.emb_matrix, axis=1)
# For the OOV words which have zero vectors,
# set the norm to 1.0 so we don't have divide-by-zero errors
for idx in oov_indices:
self.emb_matrix_norm[idx] = 1.0
示例11: _get_embtype
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import GloVe [as 别名]
def _get_embtype(self, emb_type):
# set up preinitialized embeddings
try:
import torchtext.vocab as vocab
except ImportError as ex:
print('Please install torch text with `pip install torchtext`')
raise ex
pretrained_dim = 300
if emb_type.startswith('glove'):
if 'twitter' in emb_type:
init = 'glove-twitter'
name = 'twitter.27B'
pretrained_dim = 200
else:
init = 'glove'
name = '840B'
embs = vocab.GloVe(
name=name, dim=pretrained_dim,
cache=modelzoo_path(self.opt.get('datapath'),
'models:glove_vectors'))
elif emb_type.startswith('fasttext_cc'):
init = 'fasttext_cc'
from parlai.zoo.fasttext_cc_vectors.build import download
embs = download(self.opt.get('datapath'))
elif emb_type.startswith('fasttext'):
init = 'fasttext'
from parlai.zoo.fasttext_vectors.build import download
embs = download(self.opt.get('datapath'))
else:
raise RuntimeError('embedding type {} not implemented. check arg, '
'submit PR to this function, or override it.'
''.format(emb_type))
return embs, init
示例12: _initialize_glove
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import GloVe [as 别名]
def _initialize_glove(self) -> torch.Tensor:
r"""
Initialize embeddings of all the tokens in a given
:class:`~allennlp.data.vocabulary.Vocabulary` by their GloVe vectors.
Extended Summary
----------------
It is recommended to train an :class:`~updown.models.updown_captioner.UpDownCaptioner` with
frozen word embeddings when one wishes to perform Constrained Beam Search decoding during
inference. This is because the constraint words may not appear in caption vocabulary (out of
domain), and their embeddings will never be updated during training. Initializing with frozen
GloVe embeddings is helpful, because they capture more meaningful semantics than randomly
initialized embeddings.
Returns
-------
torch.Tensor
GloVe Embeddings corresponding to tokens.
"""
glove = GloVe(name="42B", dim=300)
glove_vectors = torch.zeros(self._vocabulary.get_vocab_size(), 300)
for word, i in self._vocabulary.get_token_to_index_vocabulary().items():
if word in glove.stoi:
glove_vectors[i] = glove.vectors[glove.stoi[word]]
elif word != self._pad_index:
# Initialize by random vector.
glove_vectors[i] = 2 * torch.randn(300) - 1
return glove_vectors
示例13: load_dataset
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import GloVe [as 别名]
def load_dataset(test_sen=None):
"""
tokenizer : Breaks sentences into a list of words. If sequential=False, no tokenization is applied
Field : A class that stores information about the way of preprocessing
fix_length : An important property of TorchText is that we can let the input to be variable length, and TorchText will
dynamically pad each sequence to the longest sequence in that "batch". But here we are using fi_length which
will pad each sequence to have a fix length of 200.
build_vocab : It will first make a vocabulary or dictionary mapping all the unique words present in the train_data to an
idx and then after it will use GloVe word embedding to map the index to the corresponding word embedding.
vocab.vectors : This returns a torch tensor of shape (vocab_size x embedding_dim) containing the pre-trained word embeddings.
BucketIterator : Defines an iterator that batches examples of similar lengths together to minimize the amount of padding needed.
"""
tokenize = lambda x: x.split()
TEXT = data.Field(sequential=True, tokenize=tokenize, lower=True, include_lengths=True, batch_first=True, fix_length=200)
LABEL = data.LabelField(tensor_type=torch.FloatTensor)
train_data, test_data = datasets.IMDB.splits(TEXT, LABEL)
TEXT.build_vocab(train_data, vectors=GloVe(name='6B', dim=300))
LABEL.build_vocab(train_data)
word_embeddings = TEXT.vocab.vectors
print ("Length of Text Vocabulary: " + str(len(TEXT.vocab)))
print ("Vector size of Text Vocabulary: ", TEXT.vocab.vectors.size())
print ("Label Length: " + str(len(LABEL.vocab)))
train_data, valid_data = train_data.split() # Further splitting of training_data to create new training_data & validation_data
train_iter, valid_iter, test_iter = data.BucketIterator.splits((train_data, valid_data, test_data), batch_size=32, sort_key=lambda x: len(x.text), repeat=False, shuffle=True)
'''Alternatively we can also use the default configurations'''
# train_iter, test_iter = datasets.IMDB.iters(batch_size=32)
vocab_size = len(TEXT.vocab)
return TEXT, vocab_size, word_embeddings, train_iter, valid_iter, test_iter
示例14: loadData
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import GloVe [as 别名]
def loadData(opt):
if not opt.from_torchtext:
import dataHelper as helper
return helper.loadData(opt)
device = 0 if torch.cuda.is_available() else -1
TEXT = data.Field(lower=True, include_lengths=True, batch_first=True,fix_length=opt.max_seq_len)
LABEL = data.Field(sequential=False)
if opt.dataset=="imdb":
train, test = datasets.IMDB.splits(TEXT, LABEL)
elif opt.dataset=="sst":
train, val, test = datasets.SST.splits( TEXT, LABEL, fine_grained=True, train_subtrees=True,
filter_pred=lambda ex: ex.label != 'neutral')
elif opt.dataset=="trec":
train, test = datasets.TREC.splits(TEXT, LABEL, fine_grained=True)
else:
print("does not support this datset")
TEXT.build_vocab(train, vectors=GloVe(name='6B', dim=300))
LABEL.build_vocab(train)
# print vocab information
print('len(TEXT.vocab)', len(TEXT.vocab))
print('TEXT.vocab.vectors.size()', TEXT.vocab.vectors.size())
train_iter, test_iter = data.BucketIterator.splits((train, test), batch_size=opt.batch_size,device=device,repeat=False,shuffle=True)
opt.label_size= len(LABEL.vocab)
opt.vocab_size = len(TEXT.vocab)
opt.embedding_dim= TEXT.vocab.vectors.size()[1]
opt.embeddings = TEXT.vocab.vectors
return train_iter, test_iter
示例15: embed_sent
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import GloVe [as 别名]
def embed_sent(self, sent, rem_first_sv=True):
"""
Produce a Arora-style sentence embedding for a given sentence.
Inputs:
sent: tokenized sentence; a list of strings
rem_first_sv: If True, remove the first singular value when you compute the
sentence embddings. Otherwise, don't remove it.
Returns:
sent_emb: tensor length glove_dim, or None.
If sent_emb is None, that's because all of the words were OOV for GloVe.
"""
# If we haven't loaded the torchtext GloVe embeddings, do so
if self.tt_embs is None:
self.get_glove_embs()
# Lookup glove embeddings for words
tokens = [t for t in sent if t in self.tt_embs.stoi] # in-vocab tokens
# glove_oov_tokens = [t for t in sent if t not in self.tt_embs.stoi]
# if len(glove_oov_tokens)>0:
# print("WARNING: tokens OOV for glove: ", glove_oov_tokens)
if len(tokens) == 0:
print(
'WARNING: tried to embed utterance %s but all tokens are OOV for '
'GloVe. Returning embedding=None' % sent
)
return None
word_embs = [
self.tt_embs.vectors[self.tt_embs.stoi[t]] for t in tokens
] # list of torch Tensors shape (glove_dim)
# Get unigram probabilities for the words. If we don't have a word in word2prob,
# assume it's as rare as the rarest word in word2prob.
unigram_probs = [
self.word2prob[t] if t in self.word2prob else self.min_word_prob
for t in tokens
] # list of floats
# word2prob_oov_tokens = [t for t in tokens if t not in self.word2prob]
# if len(word2prob_oov_tokens)>0:
# print('WARNING: tokens OOV for word2prob, so assuming they are '
# 'maximally rare: ', word2prob_oov_tokens)
# Calculate the weighted average of the word embeddings
smooth_inverse_freqs = [
self.arora_a / (self.arora_a + p) for p in unigram_probs
] # list of floats
sent_emb = sum(
[word_emb * wt for (word_emb, wt) in zip(word_embs, smooth_inverse_freqs)]
) / len(
word_embs
) # torch Tensor shape (glove_dim)
# Remove the first singular value from sent_emb
if rem_first_sv:
sent_emb = remove_first_sv(sent_emb, self.first_sv)
return sent_emb