Python word2vec.Vocab方法代码示例

# 需要导入模块: from gensim.models import word2vec [as 别名]
# 或者: from gensim.models.word2vec import Vocab [as 别名]
def add_word(self, word, parent_word, emb, cur_index):
        fake_vocab_size = int(1e7)
        word_index = len(self.vocab)
        inner_node_index = word_index - 1
        parent_index = self.vocab[parent_word].index

        # add in the left subtree
        if word != parent_word:
            self.vocab[word] = Vocab(index=word_index, count=fake_vocab_size-word_index,sample_int=(2**32))
            if emb is not None:
                self.syn0[cur_index] = emb
            else:
                self.syn0[cur_index] = self.syn0[parent_index]
            # the node in the coarsened graph serves as an inner node now
            self.index2word.append(word)
            self.vocab[word].code = array(list(self.vocab[parent_word].code) + [0], dtype=uint8)
            self.vocab[word].point = array(list(self.vocab[parent_word].point) + [inner_node_index], dtype=uint32)
            self.inner_node_index_map[parent_word] = inner_node_index
        else:
            if emb is not None:
                self.syn0[parent_index] = emb
            self.vocab[word].code = array(list(self.vocab[word].code) + [1], dtype=uint8)
            self.vocab[word].point = array(list(self.vocab[word].point) + [self.inner_node_index_map[word]], dtype=uint32) 

# 需要导入模块: from gensim.models import word2vec [as 别名]
# 或者: from gensim.models.word2vec import Vocab [as 别名]
def build_vocab(self, corpus):
        """
        Build vocabulary from a sequence of sentences or from a frequency dictionary, if one was provided.
        """
        if self.vocabulary_counts != None:
          logger.debug("building vocabulary from provided frequency map")
          vocab = self.vocabulary_counts
        else:
          logger.debug("default vocabulary building")
          super(Skipgram, self).build_vocab(corpus)
          return

        # assign a unique index to each word
        self.vocab, self.index2word = {}, []

        for word, count in vocab.iteritems():
            v = Vocab()
            v.count = count
            if v.count >= self.min_count:
                v.index = len(self.vocab)
                self.index2word.append(word)
                self.vocab[word] = v

        logger.debug("total %i word types after removing those with count<%s" % (len(self.vocab), self.min_count))

        if self.hs:
            # add info about each word's Huffman encoding
            self.create_binary_tree()
        if self.negative:
            # build the table for drawing random words (for negative sampling)
            self.make_table()
        # precalculate downsampling thresholds
        self.precalc_sampling()
        self.reset_weights() 

# 需要导入模块: from gensim.models import word2vec [as 别名]
# 或者: from gensim.models.word2vec import Vocab [as 别名]
def __init__(self, pathtomapping, pathtovectors, pathtocounts="", initkeys=()):
        """
        SPPMI model equivalent to a gensim word2vec model.

        :param pathtomapping:
        :param pathtovectors:
        :param pathtocounts:
        :param initkeys:
        :return:
        """

        super(SPPMIModel, self).__init__()
        self.word2index = json.load(open(pathtomapping))
        self.index2word = {v: k for k, v in self.word2index.items()}
        self.word_vectors = self._load_sparse(pathtovectors)

        self.vocab = {}

        self.fast_table = {k: {} for k in initkeys}

        if pathtocounts:

            counts = json.load(open(pathtocounts))

            for w, idx in self.word2index.items():
                v = Vocab(count=counts[w], index=idx)
                self.vocab[w] = v 

# 需要导入模块: from gensim.models import word2vec [as 别名]
# 或者: from gensim.models.word2vec import Vocab [as 别名]
def load_word2vec_format(fname, fvocab=None, binary=False, norm_only=True, encoding='utf8'):
    """
    Load the input-hidden weight matrix from the original C word2vec-tool format.

    Note that the information stored in the file is incomplete (the binary tree is missing),
    so while you can query for word similarity etc., you cannot continue training
    with a model loaded this way.

    `binary` is a boolean indicating whether the data is in binary word2vec format.
    `norm_only` is a boolean indicating whether to only store normalised word2vec vectors in memory.
    Word counts are read from `fvocab` filename, if set (this is the file generated
    by `-save-vocab` flag of the original C tool).

    If you trained the C model using non-utf8 encoding for words, specify that
    encoding in `encoding`.

    """
    counts = None
    if fvocab is not None:
        logger.info("loading word counts from %s" % (fvocab))
        counts = {}
        with utils.smart_open(fvocab) as fin:
            for line in fin:
                word, count = utils.to_unicode(line).strip().split()
                counts[word] = int(count)

    logger.info("loading projection weights from %s" % (fname))
    with utils.smart_open(fname) as fin:
        header = utils.to_unicode(fin.readline(), encoding=encoding)
        vocab_size, vector_size = map(int, header.split())  # throws for invalid file format
        result = Word2Vec(size=vector_size)
        result.syn0 = zeros((vocab_size, vector_size), dtype=REAL)
        if binary:
            binary_len = dtype(REAL).itemsize * vector_size
            for line_no in xrange(vocab_size):
                # mixed text and binary: read text first, then binary
                word = []
                while True:
                    ch = fin.read(1)
                    if ch == b' ':
                        break
                    if ch != b'\n':  # ignore newlines in front of words (some binary files have)

                        word.append(ch)
                try:
                    word = utils.to_unicode(b''.join(word), encoding=encoding)
                except UnicodeDecodeError, e:
                    logger.warning("Couldn't convert whole word to unicode: trying to convert first %d bytes only ..." % e.start)
                    word = utils.to_unicode(b''.join(word[:e.start]), encoding=encoding)
                    logger.warning("... first %d bytes converted to '%s'" % (e.start, word))

                if counts is None:
                    result.vocab[word] = Vocab(index=line_no, count=vocab_size - line_no)
                elif word in counts:
                    result.vocab[word] = Vocab(index=line_no, count=counts[word])
                else:
                    logger.warning("vocabulary file is incomplete")
                    result.vocab[word] = Vocab(index=line_no, count=None)
                result.index2word.append(word)
                result.syn0[line_no] = fromstring(fin.read(binary_len), dtype=REAL)
        else: