Python Space.build方法代码示例

# 需要导入模块: from space import Space [as 别名]
# 或者: from space.Space import build [as 别名]
    def build_src_wrapper(self, source_file, test_wpairs):
        """
        In the _source_ space, we only need to load vectors for the words in
        test.  Semantic spaces may contain additional words.
        All words in the _target_ space are used as the search space
        """
        source_words = set(test_wpairs.iterkeys())
        if self.additional:
            #read all the words in the space
            lexicon = set(np.loadtxt(source_file, skiprows=1, dtype=str,
                                        comments=None, usecols=(0,)).flatten())
            #the max number of additional+test elements is bounded by the size
            #of the lexicon
            self.additional = min(self.additional, len(lexicon) - len(source_words))
            random.seed(100)
            logging.info("Sampling {} additional elements".format(self.additional))
            # additional lexicon:
            lexicon = random.sample(list(lexicon.difference(source_words)),
                                    self.additional)

            #load the source space
            source_sp = Space.build(source_file,
                                    lexicon=source_words.union(set(lexicon)),
                                    max_rows=1000)
        else:
            source_sp = Space.build(source_file, lexicon=source_words,
                                    max_rows=1000) 
        source_sp.normalize()
        return source_sp

# 需要导入模块: from space import Space [as 别名]
# 或者: from space.Space import build [as 别名]
def train_wrapper(seed_fn, source_fn, target_fn, reverse=False, mx_path=None,
                  train_size=5000):
    logging.info("Training...")
    seed_trans = read_dict(seed_fn, reverse=reverse)

    #we only need to load the vectors for the words in the training data
    #semantic spaces contain additional words
    source_words = set(seed_trans.iterkeys())
    target_words = set().union(*seed_trans.itervalues())

    source_sp = Space.build(source_fn, lexicon=source_words)
    source_sp.normalize()

    target_sp = Space.build(target_fn, lexicon=target_words)
    target_sp.normalize()

    logging.info("Learning the translation matrix")
    tm, used_for_train = train_tm(source_sp, target_sp, seed_trans, train_size)

    mx_path = default_output_fn(mx_path, seed_fn, source_fn, target_fn,)
    logging.info("Saving the translation matrix to {}".format(mx_path))
    np.save('{}.npy'.format(mx_path), tm)
    pickle.dump(used_for_train, open('{}.train_wds'.format(mx_path),
                                     mode='w'))

    return tm, used_for_train

# 需要导入模块: from space import Space [as 别名]
# 或者: from space.Space import build [as 别名]
def train_translation_matrix(source_file, target_file, dict_file, out_file):
    """Trains a transltion matrix between the source and target languages, using the words in dict_file as anchor
    points and writing the translation matrix to out_file

    Note that the source language file and target language file must be in the word2vec C ASCII format

    :param source_file: The name of the source language file
    :param target_file: The name of the target language file
    :param dict_file: The name of the file with the bilingual dictionary
    :param out_file: The name of the file to write the translation matrix to
    """
    log.info("Reading the training data")
    train_data = read_dict(dict_file)

    #we only need to load the vectors for the words in the training data
    #semantic spaces contain additional words
    source_words, target_words = zip(*train_data)

    log.info("Reading: %s" % source_file)
    source_sp = Space.build(source_file, set(source_words))
    source_sp.normalize()

    log.info("Reading: %s" % target_file)
    target_sp = Space.build(target_file, set(target_words))
    target_sp.normalize()

    log.debug('Words in the source space: %s' % source_sp.row2id)
    log.debug('Words in the target space: %s' % target_sp.row2id)

    log.info("Learning the translation matrix")
    log.info("Training data: %s" % str(train_data))
    tm = train_tm(source_sp, target_sp, train_data)

    log.info("Printing the translation matrix")
    np.savetxt(out_file, tm)

# 需要导入模块: from space import Space [as 别名]
# 或者: from space.Space import build [as 别名]
    def test_wrapper(self):
        self.load_tr_mx()

        logging.info('The denominator of precision {} OOV words'.format(
                          'includes' if self.args.coverage 
                          else "doesn't include"))
        test_wpairs = read_dict(self.args.seed_fn, reverse=self.args.reverse,
                                needed=1000 if self.args.coverage else -1,
                                exclude=self.exclude_from_test)

        source_sp = self.build_src_wrapper(self.args.source_fn, test_wpairs)

        target_sp = Space.build(self.args.target_fn)
        target_sp.normalize()

        test_wpairs, _ = get_invocab_trans(source_sp, target_sp,
                                              test_wpairs, needed=1000)

        """
        #turn test data into a dictionary (a word can have mutiple translation)
        gold = collections.defaultdict(set, test_wpairs)
        for sr, tg in test_wpairs:
            gold[sr].add(tg)
            """

        logging.info(
            "Mapping all the elements loaded in the source space")
        mapped_source_sp = apply_tm(source_sp, self.tr_mx)
        if hasattr(self.args, 'mapped_vecs') and self.args.mapped_vecs:
            logging.info("Printing mapped vectors: %s" % self.args.mapped_vecs)
            np.savetxt("%s.vecs.txt" % self.args.mapped_vecs, mapped_source_sp.mat)
            np.savetxt("%s.wds.txt" %
                       self.args.mapped_vecs, mapped_source_sp.id2word, fmt="%s")

        return score(mapped_source_sp, target_sp, test_wpairs, self.additional)

# 需要导入模块: from space import Space [as 别名]
# 或者: from space.Space import build [as 别名]
    def build(cls, core_space, **kwargs):
        """
        Reads in data files and extracts the data to construct a semantic space.

        If the data is read in dense format and no columns are provided,
        the column indexing structures are set to empty.

        Args:
            data: file containing the counts
            format: format on the input data file: one of sm/dm
            rows: file containing the row elements. Optional, if not provided,
                extracted from the data file.
            cols: file containing the column elements

        Returns:
            A semantic space build from the input data files.

        Raises:
            ValueError: if one of data/format arguments is missing.
                        if cols is missing and format is "sm"
                        if the input columns provided are not consistent with
                        the shape of the matrix (for "dm" format)

        """

        sp = Space.build(**kwargs)

        mat = sp._cooccurrence_matrix
        id2row = sp.id2row
        row2id = sp.row2id
        return PeripheralSpace(core_space, mat, id2row, row2id)

# 需要导入模块: from space import Space [as 别名]
# 或者: from space.Space import build [as 别名]
    print "Loading the translation matrix"
    tm = np.loadtxt(tm_file)

    print "Reading the test data"
    test_data = read_dict(test_file)

    # in the _source_ space, we only need to load vectors for the words in test.
    # semantic spaces may contain additional words, ALL words in the _target_
    # space are used as the search space
    source_words, _ = zip(*test_data)
    source_words = set(source_words)

    print "Reading: %s" % source_file
    if not additional:
        source_sp = Space.build(source_file, source_words)
    else:
        # read all the words in the space
        lexicon = set(np.loadtxt(source_file, skiprows=1, dtype=str,
                                 comments=None, usecols=(0,)).flatten())
        # the max number of additional+test elements is bounded by the size
        # of the lexicon
        additional = min(additional, len(lexicon) - len(source_words))
        # we sample additional elements that are not already in source_words
        random.seed(100)
        lexicon = random.sample(list(lexicon.difference(source_words)), additional)

        # load the source space
        source_sp = Space.build(source_file, source_words.union(set(lexicon)))

    source_sp.normalize()

# 需要导入模块: from space import Space [as 别名]
# 或者: from space.Space import build [as 别名]
        target_file = argv[2]
	dict_file = argv[0]
    else:
	print str(err)
	usage(1)


    print "Reading the training data"
    train_data = read_dict(dict_file)

    #we only need to load the vectors for the words in the training data
    #semantic spaces contain additional words
    source_words, target_words = zip(*train_data)

    print "Reading: %s" % source_file
    source_sp = Space.build(source_file, set(source_words))
    source_sp.normalize()

    print "Reading: %s" % target_file
    target_sp = Space.build(target_file, set(target_words))
    target_sp.normalize()

    print "Learning the translation matrix"
    print "Training data: %s" % str(train_data)
    tm = train_tm(source_sp, target_sp, train_data)

    print "Printing the translation matrix"
    np.savetxt("%s.txt" % out_file, tm)


if __name__ == '__main__':