Python fastText.load_model方法代码示例

# 需要导入模块: import fastText [as 别名]
# 或者: from fastText import load_model [as 别名]
def __init__(self, path=data_path):
        self._clf = [
            fastText.load_model(os.path.join(path, 'clf0.bin')),
            fastText.load_model(os.path.join(path, 'clf1.bin')),
            fastText.load_model(os.path.join(path, 'clf2.bin')),
        ]
        self._zh_chars = re.compile(r'[^\u4e00-\u9fff]+')
        self._id2name = dict()
        with open(os.path.join(data_path, 'nsfc_subject.csv'), encoding='utf-8') as f:
            for line in f:
                _id, _name = line[:-1].split(',') # encode csv
                self._id2name[_id] = _name 

# 需要导入模块: import fastText [as 别名]
# 或者: from fastText import load_model [as 别名]
def __init__(self):
        self._model = fastText.load_model(os.path.join(data_path, 'model_aminer'))
        self._words = self._model.get_labels()
        self._index_mat = joblib.load(os.path.join(data_path, 'index_mat.pkl'))
        self._id2person = json.load(open(os.path.join(data_path, 'pid_list.json'), encoding='utf-8'))
        self.ac = ACAutomaton(self._words)
        self.base_url = 'http://www.aminer.cn/profile/{}' 

# 需要导入模块: import fastText [as 别名]
# 或者: from fastText import load_model [as 别名]
def load_fasttext_model(path):
    """
    Load a binarized fastText model.
    """
    try:
        import fastText
    except ImportError:
        raise Exception("Unable to import fastText. Please install fastText for Python: "
                        "https://github.com/facebookresearch/fastText")
    return fastText.load_model(path) 

# 需要导入模块: import fastText [as 别名]
# 或者: from fastText import load_model [as 别名]
def make_embeddings_simple_in_memory(self, name="fasttext-crawl"):
        nbWords = 0
        print('loading embeddings...')
        begin = True
        description = self._get_description(name)
        if description is not None:
            embeddings_path = description["path"]
            embeddings_type = description["type"]
            self.lang = description["lang"]
            print("path:", embeddings_path)
            if self.extension == 'bin':
                self.model = fastText.load_model(embeddings_path)
                nbWords = len(self.model.get_words())
                self.embed_size = self.model.get_dimension()
            else:
                with open(embeddings_path, encoding='utf8') as f:
                    for line in f:
                        line = line.strip()
                        line = line.split(' ')
                        if begin:
                            begin = False
                            nb_words, embed_size = fetch_header_if_available(line)

                            # we parse the header
                            if nb_words > 0 and embed_size > 0:
                                nbWords = nb_words
                                self.embed_size = embed_size
                                continue

                        word = line[0]
                        #if embeddings_type == 'glove':
                        vector = np.array([float(val) for val in line[1:len(line)]], dtype='float32')
                        #else:
                        #    vector = np.array([float(val) for val in line[1:len(line)-1]], dtype='float32')
                        if self.embed_size == 0:
                            self.embed_size = len(vector)
                        self.model[word] = vector
                if nbWords == 0:
                    nbWords = len(self.model)
            print('embeddings loaded for', nbWords, "words and", self.embed_size, "dimensions") 

# 需要导入模块: import fastText [as 别名]
# 或者: from fastText import load_model [as 别名]
def before_request():
    g.ft_model = fastText.load_model(app.config["FT_SERVER_MODEL_PATH"]) 

# 需要导入模块: import fastText [as 别名]
# 或者: from fastText import load_model [as 别名]
def load(self, embedding_fname, embedding_url=None, *args, **kwargs):
        """
        Method initializes dict of embeddings from file
        Args:
            fname: file name

        Returns:
            Nothing
        """

        if not embedding_fname:
            raise RuntimeError('Please, provide path to model')
        fasttext_model_file = embedding_fname

        if not Path(fasttext_model_file).is_file():
            emb_path = embedding_url
            if not emb_path:
                raise RuntimeError('Fasttext model file does not exist locally. URL does not contain  fasttext model file')
            embedding_fname = Path(fasttext_model_file).name
            try:
                download(dest_file_path=fasttext_model_file, source_url=embedding_url)
            except Exception as e:
                raise RuntimeError('Looks like the `EMBEDDINGS_URL` variable is set incorrectly', e)

        if self.module == "fastText":
            import fastText
            self.fasttext_model = fastText.load_model(fasttext_model_file)
        if self.module == "fasttext":
            import fasttext
            self.fasttext_model = fasttext.load_model(fasttext_model_file)
        return