Java JapaneseTokenizer类代码示例

import org.codelibs.neologd.ipadic.lucene.analysis.ja.JapaneseTokenizer; //导入依赖的package包/类
TokenizerWrapper() {
    super();

    tokenizerTimestamp = dictionaryTimestamp;
    tokenizer = new JapaneseTokenizer(userDictionary, discartPunctuation, mode);

    try {
        Field attributesField = getAccessibleField(AttributeSource.class, "attributes");
        final Object attributesObj = attributesField.get(tokenizer);
        attributesField.set(this, attributesObj);

        Field attributeImplsField = getAccessibleField(AttributeSource.class, "attributeImpls");
        final Object attributeImplsObj = attributeImplsField.get(tokenizer);
        attributeImplsField.set(this, attributeImplsObj);

        Field currentStateField = getAccessibleField(AttributeSource.class, "currentState");
        final Object currentStateObj = currentStateField.get(tokenizer);
        currentStateField.set(this, currentStateObj);
    } catch (final Exception e) {
        throw new IllegalStateException(
                "Failed to update the tokenizer.", e);
    }
}
 

import org.codelibs.neologd.ipadic.lucene.analysis.ja.JapaneseTokenizer; //导入依赖的package包/类
public KuromojiAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) {
    super(indexSettings, name, settings);
    final Set<?> stopWords =
            Analysis.parseStopWords(env, indexSettings.getIndexVersionCreated(), settings, JapaneseAnalyzer.getDefaultStopSet());
    final JapaneseTokenizer.Mode mode = KuromojiTokenizerFactory.getMode(settings);
    final UserDictionary userDictionary = KuromojiTokenizerFactory.getUserDictionary(env, settings);
    analyzer = new JapaneseAnalyzer(userDictionary, mode, CharArraySet.copy(stopWords), JapaneseAnalyzer.getDefaultStopTags());
}
 

import org.codelibs.neologd.ipadic.lucene.analysis.ja.JapaneseTokenizer; //导入依赖的package包/类
public static JapaneseTokenizer.Mode getMode(Settings settings) {
    JapaneseTokenizer.Mode mode = JapaneseTokenizer.DEFAULT_MODE;
    String modeSetting = settings.get("mode", null);
    if (modeSetting != null) {
        if ("search".equalsIgnoreCase(modeSetting)) {
            mode = JapaneseTokenizer.Mode.SEARCH;
        } else if ("normal".equalsIgnoreCase(modeSetting)) {
            mode = JapaneseTokenizer.Mode.NORMAL;
        } else if ("extended".equalsIgnoreCase(modeSetting)) {
            mode = JapaneseTokenizer.Mode.EXTENDED;
        }
    }
    return mode;
}
 

import org.codelibs.neologd.ipadic.lucene.analysis.ja.JapaneseTokenizer; //导入依赖的package包/类
@Override
public Tokenizer create() {
    return new JapaneseTokenizer(userDictionary, discartPunctuation, mode);
}
 

import org.codelibs.neologd.ipadic.lucene.analysis.ja.JapaneseTokenizer; //导入依赖的package包/类
public ReloadableKuromojiTokenizerFactory(IndexSettings indexSettings, Environment env, String name, Settings settings) {
    super(indexSettings, name, settings);
    this.env = env;
    this.settings = settings;
    mode = KuromojiTokenizerFactory.getMode(settings);
    userDictionary = KuromojiTokenizerFactory.getUserDictionary(env,
            settings);
    discartPunctuation = settings.getAsBoolean("discard_punctuation", true);

    inputPendingField = getAccessibleField(Tokenizer.class, "inputPending");
    userDictionaryField = getAccessibleField(JapaneseTokenizer.class, "userDictionary");
    userFSTField = getAccessibleField(JapaneseTokenizer.class, "userFST");
    userFSTReaderField = getAccessibleField(JapaneseTokenizer.class, "userFSTReader");
    dictionaryMapField = getAccessibleField(JapaneseTokenizer.class, "dictionaryMap");

    dictionaryTimestamp = System.currentTimeMillis();

    final String monitoringFilePath = settings.get("user_dictionary");
    if (monitoringFilePath != null) {
        Path path = env.configFile().resolve(monitoringFilePath);

        try {
            final File file = path.toFile();
            if (file.exists()) {
                reloadableFile = file;
                dictionaryTimestamp = reloadableFile.lastModified();

                reloadInterval = settings.getAsTime("reload_interval",
                        TimeValue.timeValueMinutes(1)).getMillis();

                if (VERBOSE) {
                    System.out.println("Check "
                            + reloadableFile.getAbsolutePath()
                            + " (interval: " + reloadInterval + "ms)");
                }
            }
        } catch (Exception e) {
            throw new IllegalArgumentException(
                    "Could not access " + monitoringFilePath, e);
        }
    }

}
 

import org.codelibs.neologd.ipadic.lucene.analysis.ja.JapaneseTokenizer; //导入依赖的package包/类
public NeologdJapaneseTokenizer() {
    this.tokenizer = new JapaneseTokenizer(new StringReader(""), null, false, JapaneseTokenizer.Mode.NORMAL);
}