Java Predefine类代码示例

import com.hankcs.hanlp.utility.Predefine; //导入依赖的package包/类
@Override
protected void roleTag(List<List<IWord>> sentenceList)
{
    logger.info("开始标注");
    int i = 0;
    for (List<IWord> wordList : sentenceList)
    {
        logger.info(++i + " / " + sentenceList.size());
        for (IWord word : wordList)
        {
            Precompiler.compile(word);  // 编译为等效字符串
        }
        LinkedList<IWord> wordLinkedList = (LinkedList<IWord>) wordList;
        wordLinkedList.addFirst(new Word(Predefine.TAG_BIGIN, Nature.begin.toString()));
        wordLinkedList.addLast(new Word(Predefine.TAG_END, Nature.end.toString()));
    }
}
 

import com.hankcs.hanlp.utility.Predefine; //导入依赖的package包/类
/**
 * 从dat文件中加载排好的trie
 *
 * @param path
 * @return
 */
private static boolean loadDat(String path)
{
    try
    {
        ByteArray byteArray = ByteArray.createByteArray(path + Predefine.BIN_EXT);
        if (byteArray == null) return false;

        int size = byteArray.nextInt();
        Integer[] value = new Integer[size];
        for (int i = 0; i < size; i++)
        {
            value[i] = byteArray.nextInt();
        }
        if (!trie.load(byteArray, value)) return false;
    }
    catch (Exception e)
    {
        return false;
    }

    return true;
}
 

import com.hankcs.hanlp.utility.Predefine; //导入依赖的package包/类
/**
 * 读取词典
 * @param path
 * @param trie
 * @param reverse 是否将其翻转
 * @return
 */
static boolean load(String path, AhoCorasickDoubleArrayTrie<String> trie, boolean reverse)
{
    String datPath = path;
    if (reverse)
    {
        datPath += Predefine.REVERSE_EXT;
    }
    if (loadDat(datPath, trie)) return true;
    // 从文本中载入并且尝试生成dat
    TreeMap<String, String> map = new TreeMap<String, String>();
    if (!load(map, reverse, path)) return false;
    logger.info("正在构建AhoCorasickDoubleArrayTrie，来源：" + path);
    trie.build(map);
    logger.info("正在缓存双数组" + datPath);
    saveDat(datPath, trie, map.entrySet());
    return true;
}
 

import com.hankcs.hanlp.utility.Predefine; //导入依赖的package包/类
private static boolean load(String path)
{
    String binPath = path + Predefine.BIN_EXT;
    if (loadBin(binPath)) return true;
    CONVERT = new char[Character.MAX_VALUE + 1];
    for (int i = 0; i < CONVERT.length; i++)
    {
        CONVERT[i] = (char) i;
    }
    IOUtil.LineIterator iterator = new IOUtil.LineIterator(path);
    while (iterator.hasNext())
    {
        String line = iterator.next();
        if (line == null) return false;
        if (line.length() != 3) continue;
        CONVERT[line.charAt(0)] = CONVERT[line.charAt(2)];
    }
    logger.info("正在缓存字符正规化表到" + binPath);
    IOUtil.saveObjectTo(CONVERT, binPath);

    return true;
}
 

import com.hankcs.hanlp.utility.Predefine; //导入依赖的package包/类
/**
 * 保存dat到磁盘
 * @param map
 * @return
 */
static boolean saveDat(TreeMap<String, Character> map)
{
    try
    {
        DataOutputStream out = new DataOutputStream(IOUtil.newOutputStream(path + Predefine.VALUE_EXT));
        out.writeInt(map.size());
        for (Character character : map.values())
        {
            out.writeChar(character);
        }
        out.close();
    }
    catch (Exception e)
    {
        logger.warning("保存值" + path + Predefine.VALUE_EXT + "失败" + e);
        return false;
    }
    return trie.save(path + Predefine.TRIE_EXT);
}
 

import com.hankcs.hanlp.utility.Predefine; //导入依赖的package包/类
public String rewrite(String text)
{
    List<Term> termList = StandardTokenizer.segment(text.toCharArray());
    StringBuilder sbOut = new StringBuilder((int) (text.length() * 1.2));
    String preWord = Predefine.TAG_BIGIN;
    for (Term term : termList)
    {
        SynonymItem synonymItem = get(term.word);
        Synonym synonym;
        if (synonymItem != null && (synonym = synonymItem.randomSynonym(Type.EQUAL, preWord)) != null)
        {
            sbOut.append(synonym.realWord);
        }
        else sbOut.append(term.word);
        preWord = PosTagCompiler.compile(term.nature.toString(), term.word);
    }
    return sbOut.toString();
}
 

import com.hankcs.hanlp.utility.Predefine; //导入依赖的package包/类
public boolean load(String path)
{
    trie = new BinTrie<Byte>();
    if (loadDat(path + Predefine.TRIE_EXT)) return true;
    String line = null;
    try
    {
        BufferedReader bw = new BufferedReader(new InputStreamReader(IOUtil.newInputStream(path)));
        while ((line = bw.readLine()) != null)
        {
            trie.put(line, null);
        }
        bw.close();
    }
    catch (Exception e)
    {
        logger.warning("加载" + path + "失败，" + e);
    }
    if (!trie.save(path + Predefine.TRIE_EXT)) logger.warning("缓存" + path + Predefine.TRIE_EXT + "失败");
    return true;
}
 

import com.hankcs.hanlp.utility.Predefine; //导入依赖的package包/类
static boolean saveDat(String path, AhoCorasickDoubleArrayTrie<Pinyin[]> trie, Set<Map.Entry<String, Pinyin[]>> entrySet)
{
    try
    {
        DataOutputStream out = new DataOutputStream(IOUtil.newOutputStream(path + Predefine.BIN_EXT));
        out.writeInt(entrySet.size());
        for (Map.Entry<String, Pinyin[]> entry : entrySet)
        {
            Pinyin[] value = entry.getValue();
            out.writeInt(value.length);
            for (Pinyin pinyin : value)
            {
                out.writeInt(pinyin.ordinal());
            }
        }
        trie.save(out);
        out.close();
    }
    catch (Exception e)
    {
        logger.warning("缓存值dat" + path + "失败");
        return false;
    }

    return true;
}
 

import com.hankcs.hanlp.utility.Predefine; //导入依赖的package包/类
static boolean load(String path)
{
    trie = new DoubleArrayTrie<String>();
    if (loadDat(path + ".bi" + Predefine.BIN_EXT)) return true;
    TreeMap<String, String> map = new TreeMap<String, String>();
    for (String line : IOUtil.readLineListWithLessMemory(path))
    {
        String[] param = line.split(" ");
        if (param[0].endsWith("@"))
        {
            continue;
        }
        String dependency = param[1];
        map.put(param[0], dependency);
    }
    if (map.size() == 0) return false;
    trie.build(map);
    if (!saveDat(path, map)) logger.warning("缓存" + path + Predefine.BIN_EXT + "失败");
    return true;
}
 

import com.hankcs.hanlp.utility.Predefine; //导入依赖的package包/类
static boolean saveDat(String path, TreeMap<String, String> map)
{
    Collection<String> dependencyList = map.values();
    // 缓存值文件
    try
    {
        DataOutputStream out = new DataOutputStream(IOUtil.newOutputStream(path +  ".bi" + Predefine.BIN_EXT));
        out.writeInt(dependencyList.size());
        for (String dependency : dependencyList)
        {
            out.writeUTF(dependency);
        }
        if (!trie.save(out)) return false;
        out.close();
    }
    catch (Exception e)
    {
        logger.warning("保存失败" + e);
        return false;
    }
    return true;
}
 

import com.hankcs.hanlp.utility.Predefine; //导入依赖的package包/类
/**
 * 将属性的词性锁定为nature，此重载会降低性能
 * 
 * @param nature
 *            词性
 * @param updateWord
 *            是否更新预编译字串
 * @return 如果锁定词性在词性列表中，返回真，否则返回假
 */
public boolean confirmNature(Nature nature, boolean updateWord) {
	switch (nature) {

	case m:
		word = Predefine.TAG_NUMBER;
		break;
	case t:
		word = Predefine.TAG_TIME;
		break;
	default:
		logger.warning("没有与" + nature + "对应的case");
		break;
	}

	return confirmNature(nature);
}
 

import com.hankcs.hanlp.utility.Predefine; //导入依赖的package包/类
/**
 * 从短到长获取至多 n 条路径
 * @param n
 * @return
 */
public List<int[]> getNPaths(int n)
{
    List<int[]> result = new ArrayList<int[]>();

    n = Math.min(Predefine.MAX_SEGMENT_NUM, n);
    for (int i = 0; i < N && result.size() < n; ++i)
    {
        List<int[]> pathList = getPaths(i);
        for (int[] path : pathList)
        {
            if (result.size() == n) break;
            result.add(path);
        }
    }

    return result;
}
 

import com.hankcs.hanlp.utility.Predefine; //导入依赖的package包/类
public MaxEntDependencyParser()
{
    String path = HanLP.Config.MaxEntModelPath + Predefine.BIN_EXT;
    model = GlobalObjectPool.get(path);
    if (model != null) return;
    long start = System.currentTimeMillis();
    ByteArray byteArray = ByteArrayFileStream.createByteArrayFileStream(path);
    if (byteArray != null)
    {
        model = MaxEntModel.create(byteArray);
    }
    else
    {
        model = MaxEntModel.create(HanLP.Config.MaxEntModelPath);
    }
    if (model != null)
    {
        GlobalObjectPool.put(path, model);
    }
    String result = model == null ? "失败" : "成功";
    logger.info("最大熵依存句法模型载入" + result + "，耗时" + (System.currentTimeMillis() - start) + " ms");
}
 

import com.hankcs.hanlp.utility.Predefine; //导入依赖的package包/类
/**
 * 加载parser模型
 * @param path
 * @return
 */
public boolean load(String path)
{
    String binPath = path + Predefine.BIN_EXT;
    if (load(ByteArrayStream.createByteArrayStream(binPath))) return true;
    if (!loadTxt(path)) return false;
    try
    {
        logger.info("正在缓存" + binPath);
        DataOutputStream out = new DataOutputStream(IOUtil.newOutputStream(binPath));
        save(out);
        out.close();
    }
    catch (Exception e)
    {
        logger.warning("缓存" + binPath + "失败：\n" + TextUtility.exceptionToString(e));
    }

    return true;
}
 

import com.hankcs.hanlp.utility.Predefine; //导入依赖的package包/类
public void testLoadModelWithBiGramFeature() throws Exception
{
    String path = HanLP.Config.CRFSegmentModelPath + Predefine.BIN_EXT;
    CRFModel model = new CRFModel(new BinTrie<FeatureFunction>());
    model.load(ByteArray.createByteArray(path));

    Table table = new Table();
    String text = "人民生活进一步改善了";
    table.v = new String[text.length()][2];
    for (int i = 0; i < text.length(); i++)
    {
        table.v[i][0] = String.valueOf(text.charAt(i));
    }

    model.tag(table);
    System.out.println(table);
}