Java AnalyzeResponse.AnalyzeToken方法代码示例

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
public void testAnalyzerWithFieldOrTypeTests() throws Exception {
    assertAcked(prepareCreate("test").addAlias(new Alias("alias")));
    ensureGreen();

    client().admin().indices().preparePutMapping("test")
            .setType("document").setSource("simple", "type=text,analyzer=simple").get();

    for (int i = 0; i < 10; i++) {
        final AnalyzeRequestBuilder requestBuilder = client().admin().indices().prepareAnalyze("THIS IS A TEST");
        requestBuilder.setIndex(indexOrAlias());
        requestBuilder.setField("document.simple");
        AnalyzeResponse analyzeResponse = requestBuilder.get();
        assertThat(analyzeResponse.getTokens().size(), equalTo(4));
        AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(3);
        assertThat(token.getTerm(), equalTo("test"));
        assertThat(token.getStartOffset(), equalTo(10));
        assertThat(token.getEndOffset(), equalTo(14));
        assertThat(token.getPositionLength(), equalTo(1));
    }
}
 

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
public void testAnalyzeNormalizedKeywordField() throws IOException {
    assertAcked(prepareCreate("test").addAlias(new Alias("alias"))
        .setSettings(Settings.builder().put(indexSettings())
            .put("index.analysis.normalizer.my_normalizer.type", "custom")
            .putArray("index.analysis.normalizer.my_normalizer.filter", "lowercase"))
        .addMapping("test", "keyword", "type=keyword,normalizer=my_normalizer"));
    ensureGreen("test");

    AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "ABC").setField("keyword").get();
    assertThat(analyzeResponse.getTokens().size(), equalTo(1));
    AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0);
    assertThat(token.getTerm(), equalTo("abc"));
    assertThat(token.getStartOffset(), equalTo(0));
    assertThat(token.getEndOffset(), equalTo(3));
    assertThat(token.getPosition(), equalTo(0));
    assertThat(token.getPositionLength(), equalTo(1));

}
 

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
@Test
public void testTokenizer() throws Exception {
    AnalyzeRequest analyzeRequest = new AnalyzeRequest();
    analyzeRequest.text("My œsophagus caused a débâcle");
    /**
     * whitespace （空白字符）分词器按空白字符 —— 空格、tabs、换行符等等进行简单拆分
     * letter 分词器 ，采用另外一种策略，按照任何非字符进行拆分
     * standard 分词器使用 Unicode 文本分割算法
     */
    analyzeRequest.addTokenFilter("standard");
    analyzeRequest.addCharFilter("asciifolding");
    ActionFuture<AnalyzeResponse> analyzeResponseActionFuture =  client.admin().indices().analyze(analyzeRequest);
    List<AnalyzeResponse.AnalyzeToken> analyzeTokens =  analyzeResponseActionFuture.actionGet().getTokens();
    for (AnalyzeResponse.AnalyzeToken analyzeToken : analyzeTokens){
        System.out.println(analyzeToken.getTerm());
    }
}
 

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
/**
 * analyze the given text using the provided analyzer, return an ngram
 * @param text
 * @param analyzer
 * @return
 */
public Ngram analyze(String text, String analyzer){
    List<AnalyzeResponse.AnalyzeToken> tokens = client.admin().indices().prepareAnalyze(indexName,text).setAnalyzer(analyzer).get().getTokens();

    Ngram ngram = new Ngram();
    StringBuilder sb = new StringBuilder();
    for (int i=0;i<tokens.size();i++)
    {
        AnalyzeResponse.AnalyzeToken token = tokens.get(i);
        sb.append(token.getTerm());
        if (i!=tokens.size()-1){
            sb.append(" ");
        }
    }
    ngram.setNgram(sb.toString());
    return ngram;

}
 

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
@Test
public void test_defaultAnalyzer() {
    String text = "Fess (フェス) は「5 分で簡単に構築可能な全文検索サーバー」です。 Java 実行環境があればどの OS でも実行可能です。 Fess は Apache ライセンスで提供され、無料 (フリーソフト) でご利用いただけます。";
    SuggestAnalyzer analyzer = SuggestUtil.createDefaultAnalyzer(runner.client(), settings);
    final List<AnalyzeResponse.AnalyzeToken> tokens = analyzer.analyze(text, null);
    final List<AnalyzeResponse.AnalyzeToken> readingTokens = analyzer.analyzeAndReading(text, null);

    assertEquals(tokens.size(), readingTokens.size());
    for (int i = 0; i < tokens.size(); i++) {
        final String term = tokens.get(i).getTerm();
        final String reading = readingTokens.get(i).getTerm();
        switch (term) {
        case "fess":
            assertEquals("フェス", reading);
            break;
        case "全文検索":
            assertEquals("ゼンブンケンサク", reading);
            break;
        case "無料":
            assertEquals("ムリョウ", reading);
            break;
        default:
            break;
        }
    }
}
 

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
public void testSimpleAnalyzerTests() throws Exception {
    assertAcked(prepareCreate("test").addAlias(new Alias("alias")));
    ensureGreen();

    for (int i = 0; i < 10; i++) {
        AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "this is a test").get();
        assertThat(analyzeResponse.getTokens().size(), equalTo(4));
        AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0);
        assertThat(token.getTerm(), equalTo("this"));
        assertThat(token.getStartOffset(), equalTo(0));
        assertThat(token.getEndOffset(), equalTo(4));
        assertThat(token.getPosition(), equalTo(0));
        assertThat(token.getPositionLength(), equalTo(1));
        token = analyzeResponse.getTokens().get(1);
        assertThat(token.getTerm(), equalTo("is"));
        assertThat(token.getStartOffset(), equalTo(5));
        assertThat(token.getEndOffset(), equalTo(7));
        assertThat(token.getPosition(), equalTo(1));
        assertThat(token.getPositionLength(), equalTo(1));
        token = analyzeResponse.getTokens().get(2);
        assertThat(token.getTerm(), equalTo("a"));
        assertThat(token.getStartOffset(), equalTo(8));
        assertThat(token.getEndOffset(), equalTo(9));
        assertThat(token.getPosition(), equalTo(2));
        assertThat(token.getPositionLength(), equalTo(1));
        token = analyzeResponse.getTokens().get(3);
        assertThat(token.getTerm(), equalTo("test"));
        assertThat(token.getStartOffset(), equalTo(10));
        assertThat(token.getEndOffset(), equalTo(14));
        assertThat(token.getPosition(), equalTo(3));
        assertThat(token.getPositionLength(), equalTo(1));
    }
}
 

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
public void testAnalyzeWithNoIndex() throws Exception {
    AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST").setAnalyzer("simple").get();
    assertThat(analyzeResponse.getTokens().size(), equalTo(4));

    analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST").setTokenizer("keyword").addTokenFilter("lowercase").get();
    assertThat(analyzeResponse.getTokens().size(), equalTo(1));
    assertThat(analyzeResponse.getTokens().get(0).getTerm(), equalTo("this is a test"));

    analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A TEST").setTokenizer("standard").addTokenFilter("lowercase").addTokenFilter("reverse").get();
    assertThat(analyzeResponse.getTokens().size(), equalTo(4));
    AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0);
    assertThat(token.getTerm(), equalTo("siht"));
    token = analyzeResponse.getTokens().get(1);
    assertThat(token.getTerm(), equalTo("si"));
    token = analyzeResponse.getTokens().get(2);
    assertThat(token.getTerm(), equalTo("a"));
    token = analyzeResponse.getTokens().get(3);
    assertThat(token.getTerm(), equalTo("tset"));

    analyzeResponse = client().admin().indices().prepareAnalyze("of course").setTokenizer("standard").addTokenFilter("stop").get();
    assertThat(analyzeResponse.getTokens().size(), equalTo(1));
    assertThat(analyzeResponse.getTokens().get(0).getTerm(), equalTo("course"));
    assertThat(analyzeResponse.getTokens().get(0).getPosition(), equalTo(1));
    assertThat(analyzeResponse.getTokens().get(0).getStartOffset(), equalTo(3));
    assertThat(analyzeResponse.getTokens().get(0).getEndOffset(), equalTo(9));
    assertThat(analyzeResponse.getTokens().get(0).getPositionLength(), equalTo(1));
}
 

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
public void testAnalyzeWithCharFilters() throws Exception {
    assertAcked(prepareCreate("test").addAlias(new Alias("alias"))
            .setSettings(Settings.builder().put(indexSettings())
                    .put("index.analysis.char_filter.custom_mapping.type", "mapping")
                    .putArray("index.analysis.char_filter.custom_mapping.mappings", "ph=>f", "qu=>q")
                    .put("index.analysis.analyzer.custom_with_char_filter.tokenizer", "standard")
                    .putArray("index.analysis.analyzer.custom_with_char_filter.char_filter", "custom_mapping")));
    ensureGreen();

    AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze("<h2><b>THIS</b> IS A</h2> <a href=\"#\">TEST</a>").setTokenizer("standard").addCharFilter("html_strip").get();
    assertThat(analyzeResponse.getTokens().size(), equalTo(4));

    analyzeResponse = client().admin().indices().prepareAnalyze("THIS IS A <b>TEST</b>").setTokenizer("keyword").addTokenFilter("lowercase").addCharFilter("html_strip").get();
    assertThat(analyzeResponse.getTokens().size(), equalTo(1));
    assertThat(analyzeResponse.getTokens().get(0).getTerm(), equalTo("this is a test"));

    analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "jeff quit phish").setTokenizer("keyword").addTokenFilter("lowercase").addCharFilter("custom_mapping").get();
    assertThat(analyzeResponse.getTokens().size(), equalTo(1));
    assertThat(analyzeResponse.getTokens().get(0).getTerm(), equalTo("jeff qit fish"));

    analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "<a href=\"#\">jeff quit fish</a>").setTokenizer("standard").addCharFilter("html_strip").addCharFilter("custom_mapping").get();
    assertThat(analyzeResponse.getTokens().size(), equalTo(3));
    AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0);
    assertThat(token.getTerm(), equalTo("jeff"));
    token = analyzeResponse.getTokens().get(1);
    assertThat(token.getTerm(), equalTo("qit"));
    token = analyzeResponse.getTokens().get(2);
    assertThat(token.getTerm(), equalTo("fish"));
}
 

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
public void testAnalyzerWithMultiValues() throws Exception {
    assertAcked(prepareCreate("test").addAlias(new Alias("alias")));
    ensureGreen();

    client().admin().indices().preparePutMapping("test")
        .setType("document").setSource("simple", "type=text,analyzer=simple,position_increment_gap=100").get();

    String[] texts = new String[]{"THIS IS A TEST", "THE SECOND TEXT"};

    final AnalyzeRequestBuilder requestBuilder = client().admin().indices().prepareAnalyze();
    requestBuilder.setText(texts);
    requestBuilder.setIndex(indexOrAlias());
    requestBuilder.setField("simple");
    AnalyzeResponse analyzeResponse = requestBuilder.get();
    assertThat(analyzeResponse.getTokens().size(), equalTo(7));
    AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(3);
    assertThat(token.getTerm(), equalTo("test"));
    assertThat(token.getPosition(), equalTo(3));
    assertThat(token.getStartOffset(), equalTo(10));
    assertThat(token.getEndOffset(), equalTo(14));
    assertThat(token.getPositionLength(), equalTo(1));

    token = analyzeResponse.getTokens().get(5);
    assertThat(token.getTerm(), equalTo("second"));
    assertThat(token.getPosition(), equalTo(105));
    assertThat(token.getStartOffset(), equalTo(19));
    assertThat(token.getEndOffset(), equalTo(25));
    assertThat(token.getPositionLength(), equalTo(1));
}
 

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
public void testDetailAnalyzeWithMultiValues() throws Exception {
    assertAcked(prepareCreate("test").addAlias(new Alias("alias")));
    ensureGreen();
    client().admin().indices().preparePutMapping("test")
        .setType("document").setSource("simple", "type=text,analyzer=simple,position_increment_gap=100").get();

    String[] texts = new String[]{"THIS IS A TEST", "THE SECOND TEXT"};
    AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze().setIndex(indexOrAlias()).setText(texts)
        .setExplain(true).setField("simple").setText(texts).execute().get();

    assertThat(analyzeResponse.detail().analyzer().getName(), equalTo("simple"));
    assertThat(analyzeResponse.detail().analyzer().getTokens().length, equalTo(7));
    AnalyzeResponse.AnalyzeToken token = analyzeResponse.detail().analyzer().getTokens()[3];

    assertThat(token.getTerm(), equalTo("test"));
    assertThat(token.getPosition(), equalTo(3));
    assertThat(token.getStartOffset(), equalTo(10));
    assertThat(token.getEndOffset(), equalTo(14));
    assertThat(token.getPositionLength(), equalTo(1));

    token = analyzeResponse.detail().analyzer().getTokens()[5];
    assertThat(token.getTerm(), equalTo("second"));
    assertThat(token.getPosition(), equalTo(105));
    assertThat(token.getStartOffset(), equalTo(19));
    assertThat(token.getEndOffset(), equalTo(25));
    assertThat(token.getPositionLength(), equalTo(1));
}
 

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
public void testAnalyzeKeywordField() throws IOException {
    assertAcked(prepareCreate("test").addAlias(new Alias("alias")).addMapping("test", "keyword", "type=keyword"));
    ensureGreen("test");

    AnalyzeResponse analyzeResponse = client().admin().indices().prepareAnalyze(indexOrAlias(), "ABC").setField("keyword").get();
    assertThat(analyzeResponse.getTokens().size(), equalTo(1));
    AnalyzeResponse.AnalyzeToken token = analyzeResponse.getTokens().get(0);
    assertThat(token.getTerm(), equalTo("ABC"));
    assertThat(token.getStartOffset(), equalTo(0));
    assertThat(token.getEndOffset(), equalTo(3));
    assertThat(token.getPosition(), equalTo(0));
    assertThat(token.getPositionLength(), equalTo(1));
}
 

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
public void testNoIndexAnalyzers() throws IOException {
    AnalyzeRequest request = new AnalyzeRequest();
    request.analyzer("standard");
    request.text("the quick brown fox");
    AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, null, registry, environment);
    List<AnalyzeResponse.AnalyzeToken> tokens = analyze.getTokens();
    assertEquals(4, tokens.size());

    request.analyzer(null);
    request.tokenizer("whitespace");
    request.addTokenFilter("lowercase");
    request.addTokenFilter("word_delimiter");
    request.text("the qu1ck brown fox");
    analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, randomBoolean() ? indexAnalyzers : null, registry, environment);
    tokens = analyze.getTokens();
    assertEquals(6, tokens.size());
    assertEquals("qu", tokens.get(1).getTerm());
    assertEquals("1", tokens.get(2).getTerm());
    assertEquals("ck", tokens.get(3).getTerm());

    request.analyzer(null);
    request.tokenizer("whitespace");
    request.addCharFilter("html_strip");
    request.addTokenFilter("lowercase");
    request.addTokenFilter("word_delimiter");
    request.text("<p>the qu1ck brown fox</p>");
    analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, randomBoolean() ? indexAnalyzers : null, registry, environment);
    tokens = analyze.getTokens();
    assertEquals(6, tokens.size());
    assertEquals("the", tokens.get(0).getTerm());
    assertEquals("qu", tokens.get(1).getTerm());
    assertEquals("1", tokens.get(2).getTerm());
    assertEquals("ck", tokens.get(3).getTerm());
    assertEquals("brown", tokens.get(4).getTerm());
    assertEquals("fox", tokens.get(5).getTerm());
}
 

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
public void testFillsAttributes() throws IOException {
    AnalyzeRequest request = new AnalyzeRequest();
    request.analyzer("standard");
    request.text("the 1 brown fox");
    AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, null, registry, environment);
    List<AnalyzeResponse.AnalyzeToken> tokens = analyze.getTokens();
    assertEquals(4, tokens.size());
    assertEquals("the", tokens.get(0).getTerm());
    assertEquals(0, tokens.get(0).getStartOffset());
    assertEquals(3, tokens.get(0).getEndOffset());
    assertEquals(0, tokens.get(0).getPosition());
    assertEquals("<ALPHANUM>", tokens.get(0).getType());

    assertEquals("1", tokens.get(1).getTerm());
    assertEquals(4, tokens.get(1).getStartOffset());
    assertEquals(5, tokens.get(1).getEndOffset());
    assertEquals(1, tokens.get(1).getPosition());
    assertEquals("<NUM>", tokens.get(1).getType());

    assertEquals("brown", tokens.get(2).getTerm());
    assertEquals(6, tokens.get(2).getStartOffset());
    assertEquals(11, tokens.get(2).getEndOffset());
    assertEquals(2, tokens.get(2).getPosition());
    assertEquals("<ALPHANUM>", tokens.get(2).getType());

    assertEquals("fox", tokens.get(3).getTerm());
    assertEquals(12, tokens.get(3).getStartOffset());
    assertEquals(15, tokens.get(3).getEndOffset());
    assertEquals(3, tokens.get(3).getPosition());
    assertEquals("<ALPHANUM>", tokens.get(3).getType());
}
 

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
public void testNonPreBuildTokenFilter() throws IOException {
    AnalyzeRequest request = new AnalyzeRequest();
    request.tokenizer("whitespace");
    request.addTokenFilter("min_hash");
    request.text("the quick brown fox");
    AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment);
    List<AnalyzeResponse.AnalyzeToken> tokens = analyze.getTokens();
    int default_hash_count = 1;
    int default_bucket_size = 512;
    int default_hash_set_size = 1;
    assertEquals(default_hash_count * default_bucket_size * default_hash_set_size, tokens.size());

}
 

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
@Test
public void testClientConnection() throws Exception {
    AnalyzeRequest analyzeRequest = new AnalyzeRequest();
    analyzeRequest.text("美女");
    ActionFuture<AnalyzeResponse> analyzeResponseActionFuture = client.admin().indices().analyze(analyzeRequest);
    List<AnalyzeResponse.AnalyzeToken> analyzeTokens =  analyzeResponseActionFuture.actionGet().getTokens();
    for (AnalyzeResponse.AnalyzeToken analyzeToken  : analyzeTokens){
        System.out.println(analyzeToken.getTerm());
    }
}