当前位置: 首页>>代码示例>>Java>>正文

Java AnalyzeResponse.getTokens方法代码示例

本文整理汇总了Java中org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.getTokens方法的典型用法代码示例。如果您正苦于以下问题：Java AnalyzeResponse.getTokens方法的具体用法？Java AnalyzeResponse.getTokens怎么用？Java AnalyzeResponse.getTokens使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse的用法示例。


在下文中一共展示了AnalyzeResponse.getTokens方法的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。

示例1: assertIncludes

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
private void assertIncludes(String input, List<String> expectedTokens, List<String> fields) throws ExecutionException, InterruptedException, IOException {
    for (String field : fields) {
        AnalyzeResponse response = client().admin().indices().prepareAnalyze(input).setField(field).setIndex("test").execute().get();
        index("test", "type", "1", field, input);
        
        // Verify all the expected tokens are in there
        List<String> tokens = new ArrayList<String>();
        for (AnalyzeToken token : response.getTokens()) {
            assertFalse(StringUtils.isEmpty(token.getTerm()));
            tokens.add(token.getTerm());
        }
        
        flush();
        refresh();
        
        for (String expectedToken : expectedTokens) {
            assertTrue(tokens.contains(expectedToken));
            SearchResponse sr = client().prepareSearch("test").setQuery(QueryBuilders.termQuery(field, expectedToken)).execute().actionGet();
            assertThat(sr.getHits().getTotalHits(), is(1L));
            sr = client().prepareSearch("test").setQuery(QueryBuilders.termQuery(field, "bogussearchterm")).execute().actionGet();
            assertThat(sr.getHits().getTotalHits(), is(0l));
        }
    }
}
开发者ID:MyPureCloud，项目名称:elasticsearch-phone，代码行数:25，代码来源:PhoneTokenizerIntegrationTest.java

示例2: testNoIndexAnalyzers

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
public void testNoIndexAnalyzers() throws IOException {
    AnalyzeRequest request = new AnalyzeRequest();
    request.analyzer("standard");
    request.text("the quick brown fox");
    AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, null, registry, environment);
    List<AnalyzeResponse.AnalyzeToken> tokens = analyze.getTokens();
    assertEquals(4, tokens.size());

    request.analyzer(null);
    request.tokenizer("whitespace");
    request.addTokenFilter("lowercase");
    request.addTokenFilter("word_delimiter");
    request.text("the qu1ck brown fox");
    analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, randomBoolean() ? indexAnalyzers : null, registry, environment);
    tokens = analyze.getTokens();
    assertEquals(6, tokens.size());
    assertEquals("qu", tokens.get(1).getTerm());
    assertEquals("1", tokens.get(2).getTerm());
    assertEquals("ck", tokens.get(3).getTerm());

    request.analyzer(null);
    request.tokenizer("whitespace");
    request.addCharFilter("html_strip");
    request.addTokenFilter("lowercase");
    request.addTokenFilter("word_delimiter");
    request.text("<p>the qu1ck brown fox</p>");
    analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, randomBoolean() ? indexAnalyzers : null, registry, environment);
    tokens = analyze.getTokens();
    assertEquals(6, tokens.size());
    assertEquals("the", tokens.get(0).getTerm());
    assertEquals("qu", tokens.get(1).getTerm());
    assertEquals("1", tokens.get(2).getTerm());
    assertEquals("ck", tokens.get(3).getTerm());
    assertEquals("brown", tokens.get(4).getTerm());
    assertEquals("fox", tokens.get(5).getTerm());
}
开发者ID:justor，项目名称:elasticsearch_my，代码行数:37，代码来源:TransportAnalyzeActionTests.java

示例3: testFillsAttributes

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
public void testFillsAttributes() throws IOException {
    AnalyzeRequest request = new AnalyzeRequest();
    request.analyzer("standard");
    request.text("the 1 brown fox");
    AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, null, registry, environment);
    List<AnalyzeResponse.AnalyzeToken> tokens = analyze.getTokens();
    assertEquals(4, tokens.size());
    assertEquals("the", tokens.get(0).getTerm());
    assertEquals(0, tokens.get(0).getStartOffset());
    assertEquals(3, tokens.get(0).getEndOffset());
    assertEquals(0, tokens.get(0).getPosition());
    assertEquals("<ALPHANUM>", tokens.get(0).getType());

    assertEquals("1", tokens.get(1).getTerm());
    assertEquals(4, tokens.get(1).getStartOffset());
    assertEquals(5, tokens.get(1).getEndOffset());
    assertEquals(1, tokens.get(1).getPosition());
    assertEquals("<NUM>", tokens.get(1).getType());

    assertEquals("brown", tokens.get(2).getTerm());
    assertEquals(6, tokens.get(2).getStartOffset());
    assertEquals(11, tokens.get(2).getEndOffset());
    assertEquals(2, tokens.get(2).getPosition());
    assertEquals("<ALPHANUM>", tokens.get(2).getType());

    assertEquals("fox", tokens.get(3).getTerm());
    assertEquals(12, tokens.get(3).getStartOffset());
    assertEquals(15, tokens.get(3).getEndOffset());
    assertEquals(3, tokens.get(3).getPosition());
    assertEquals("<ALPHANUM>", tokens.get(3).getType());
}
开发者ID:justor，项目名称:elasticsearch_my，代码行数:32，代码来源:TransportAnalyzeActionTests.java

示例4: testNonPreBuildTokenFilter

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
public void testNonPreBuildTokenFilter() throws IOException {
    AnalyzeRequest request = new AnalyzeRequest();
    request.tokenizer("whitespace");
    request.addTokenFilter("min_hash");
    request.text("the quick brown fox");
    AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment);
    List<AnalyzeResponse.AnalyzeToken> tokens = analyze.getTokens();
    int default_hash_count = 1;
    int default_bucket_size = 512;
    int default_hash_set_size = 1;
    assertEquals(default_hash_count * default_bucket_size * default_hash_set_size, tokens.size());

}
开发者ID:justor，项目名称:elasticsearch_my，代码行数:14，代码来源:TransportAnalyzeActionTests.java

示例5: customAnalyzing

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
public String customAnalyzing(String indexName, String analyzer, String str) throws InterruptedException, ExecutionException {
  String[] strList = str.toLowerCase().split(",");
  for (int i = 0; i < strList.length; i++) {
    String tmp = "";
    AnalyzeResponse r = client.admin().indices().prepareAnalyze(strList[i]).setIndex(indexName).setAnalyzer(analyzer).execute().get();
    for (AnalyzeToken token : r.getTokens()) {
      tmp += token.getTerm() + " ";
    }
    strList[i] = tmp.trim();
  }
  return String.join(",", strList);
}
开发者ID:apache，项目名称:incubator-sdap-mudrod，代码行数:13，代码来源:ESDriver.java

示例6: showAnaylzerText

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
private static void showAnaylzerText(IndicesAdminClient indicesAdminClient,String analyzerName, String text) {
    AnalyzeResponse analyzeResponse = indicesAdminClient.analyze(new AnalyzeRequest(INDEX_NAME).analyzer(analyzerName).text(text)).actionGet();
    List<AnalyzeResponse.AnalyzeToken> token=analyzeResponse.getTokens();
    for (AnalyzeResponse.AnalyzeToken analyzeToken : token) {
        log.info(analyzerName+": {}",analyzeToken.getTerm());
    }

}
开发者ID:ggj2010，项目名称:javabase，代码行数:9，代码来源:CrudDemo.java

示例7: analyze

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
/**
 * 分词-无法分词则返回空集合
 * 
 * @param analyzer
 * @param str
 * @return
 */
public static List<String> analyze(String analyzer, String str) {

    AnalyzeResponse ar = null;
    try {
        AnalyzeRequest request = new AnalyzeRequest(str).analyzer(analyzer).index(
                getCurrentValidIndex());
        ar = ESClient.getClient().admin().indices().analyze(request).actionGet();
    } catch (IndexMissingException e) {
        if (!reLoad) {
            synchronized (AnalyzeHelper.class) {
                if (!reLoad) {
                    reLoad = true;
                }
            }
        }
        return analyze(analyzer, str);
    }

    if (ar == null || ar.getTokens() == null || ar.getTokens().size() < 1) {
        return Lists.newArrayList();
    }
    List<String> analyzeTokens = Lists.newArrayList();
    for (AnalyzeToken at : ar.getTokens()) {
        analyzeTokens.add(at.getTerm());
    }
    return analyzeTokens;
}
开发者ID:hailin0，项目名称:es-service-parent，代码行数:35，代码来源:AnalyzeHelper.java

示例8: analyzeText

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
@Override
public List<String> analyzeText(final String indexAliasName, final String analyzer, final String[] tokenFilters, final String text)
{
    final List<String> tokens = new ArrayList<String>();
    final AnalyzeRequestBuilder analyzeRequestBuilder = searchClientService.getClient().admin().indices().prepareAnalyze(text);
    if (analyzer != null)
    {
        analyzeRequestBuilder.setIndex(indexAliasName);
    }
    if (analyzer != null)
    {
        analyzeRequestBuilder.setAnalyzer(analyzer);
    }
    if (tokenFilters != null)
    {
        analyzeRequestBuilder.setTokenFilters(tokenFilters);
    }
    logger.debug("Analyze request is text: {}, analyzer: {}, tokenfilters: {}", new Object[] { analyzeRequestBuilder.request().text(),
            analyzeRequestBuilder.request().analyzer(), analyzeRequestBuilder.request().tokenFilters() });
    final AnalyzeResponse analyzeResponse = analyzeRequestBuilder.get();
    try
    {
        if (analyzeResponse != null)
        {
            logger.debug("Analyze response is : {}", analyzeResponse.toXContent(jsonBuilder().startObject(), ToXContent.EMPTY_PARAMS)
                    .prettyPrint().string());
        }
    }
    catch (final IOException e)
    {
        logger.error("Error printing response.", e);
    }
    for (final AnalyzeToken analyzeToken : analyzeResponse.getTokens())
    {
        tokens.add(analyzeToken.getTerm());
    }
    return tokens;
}
开发者ID:jaibeermalik，项目名称:elasticsearch-akka，代码行数:39，代码来源:SetupIndexServiceImpl.java

示例9: analyzeText

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
@Override
public List<String> analyzeText(final String indexAliasName,
		final String analyzer, final String[] tokenFilters,
		final String text) {
	final List<String> tokens = new ArrayList<String>();
	final AnalyzeRequestBuilder analyzeRequestBuilder = searchClientService
			.getClient().admin().indices().prepareAnalyze(text);
	if (analyzer != null) {
		analyzeRequestBuilder.setIndex(indexAliasName);
	}
	if (analyzer != null) {
		analyzeRequestBuilder.setAnalyzer(analyzer);
	}
	if (tokenFilters != null) {
		analyzeRequestBuilder.setTokenFilters(tokenFilters);
	}
	logger.debug(
			"Analyze request is text: {}, analyzer: {}, tokenfilters: {}",
			new Object[] { analyzeRequestBuilder.request().text(),
					analyzeRequestBuilder.request().analyzer(),
					analyzeRequestBuilder.request().tokenFilters() });
	final AnalyzeResponse analyzeResponse = analyzeRequestBuilder.get();
	try {
		if (analyzeResponse != null) {
			logger.debug(
					"Analyze response is : {}",
					analyzeResponse
							.toXContent(jsonBuilder().startObject(),
									ToXContent.EMPTY_PARAMS).prettyPrint()
							.string());
		}
	} catch (final IOException e) {
		logger.error("Error printing response.", e);
	}
	for (final AnalyzeToken analyzeToken : analyzeResponse.getTokens()) {
		tokens.add(analyzeToken.getTerm());
	}
	return tokens;
}
开发者ID:jaibeermalik，项目名称:searchanalytics-bigdata，代码行数:40，代码来源:SetupIndexServiceImpl.java

示例10: normalize

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
@Override
public String normalize(final String text, final String... dummy) {
    final AnalyzeResponse termResponse =
            client.admin().indices().prepareAnalyze(analyzerSettings.getAnalyzerSettingsIndexName(), text)
                    .setAnalyzer(analyzerSettings.getNormalizeAnalyzerName(lang)).execute().actionGet();

    final List<AnalyzeResponse.AnalyzeToken> termTokenList = termResponse.getTokens();
    if (termTokenList.isEmpty()) {
        return text;
    }

    return termTokenList.get(0).getTerm();
}
开发者ID:codelibs，项目名称:fess-suggest，代码行数:14，代码来源:AnalyzerNormalizer.java

示例11: analyze

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
@Override
public List<AnalyzeResponse.AnalyzeToken> analyze(final String text, final String lang) {
    final AnalyzeResponse analyzeResponse =
            client.admin().indices().prepareAnalyze(analyzerSettingsIndexName, text).setAnalyzer(getContentsAnalyzerName(lang))
                    .execute().actionGet();
    return analyzeResponse.getTokens();
}
开发者ID:codelibs，项目名称:fess-suggest，代码行数:8，代码来源:AnalyzerSettings.java

示例12: analyzeAndReading

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
@Override
public List<AnalyzeResponse.AnalyzeToken> analyzeAndReading(final String text, final String lang) {
    try {
        final AnalyzeResponse analyzeResponse =
                client.admin().indices().prepareAnalyze(analyzerSettingsIndexName, text)
                        .setAnalyzer(getContentsReadingAnalyzerName(lang)).execute().actionGet();
        return analyzeResponse.getTokens();
    } catch (final IllegalArgumentException e) {
        return analyze(text, lang);
    }
}
开发者ID:codelibs，项目名称:fess-suggest，代码行数:12，代码来源:AnalyzerSettings.java

示例13: testWithIndexAnalyzers

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
public void testWithIndexAnalyzers() throws IOException {

        AnalyzeRequest request = new AnalyzeRequest();
        request.analyzer("standard");
        request.text("the quick brown fox");
        request.analyzer("custom_analyzer");
        request.text("the qu1ck brown fox");
        AnalyzeResponse analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment);
        List<AnalyzeResponse.AnalyzeToken> tokens = analyze.getTokens();
        assertEquals(4, tokens.size());

        request.analyzer("whitespace");
        request.text("the qu1ck brown fox-dog");
        analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment);
        tokens = analyze.getTokens();
        assertEquals(4, tokens.size());

        request.analyzer("custom_analyzer");
        request.text("the qu1ck brown fox-dog");
        analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment);
        tokens = analyze.getTokens();
        assertEquals(5, tokens.size());

        request.analyzer(null);
        request.tokenizer("whitespace");
        request.addTokenFilter("lowercase");
        request.addTokenFilter("wordDelimiter");
        request.text("the qu1ck brown fox-dog");
        analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment);
        tokens = analyze.getTokens();
        assertEquals(5, tokens.size());
        assertEquals("the", tokens.get(0).getTerm());
        assertEquals("qu1ck", tokens.get(1).getTerm());
        assertEquals("brown", tokens.get(2).getTerm());
        assertEquals("fox", tokens.get(3).getTerm());
        assertEquals("dog", tokens.get(4).getTerm());

        request.analyzer(null);
        request.tokenizer("trigram");
        request.addTokenFilter("synonym");
        request.text("kimchy");
        analyze = TransportAnalyzeAction.analyze(request, AllFieldMapper.NAME, null, indexAnalyzers, registry, environment);
        tokens = analyze.getTokens();
        assertEquals(2, tokens.size());
        assertEquals("sha", tokens.get(0).getTerm());
        assertEquals("hay", tokens.get(1).getTerm());
    }
开发者ID:justor，项目名称:elasticsearch_my，代码行数:48，代码来源:TransportAnalyzeActionTests.java

示例14: analyzeText

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
@Override
public List<String> analyzeText(String indexAliasName, String analyzer, String[] tokenFilters, String text)
{
    List<String> tokens = new ArrayList<String>();
    
    AnalyzeRequestBuilder analyzeRequestBuilder = searchClientService.getClient().admin().indices().prepareAnalyze(text);
    
    if(analyzer !=null)
    {
        analyzeRequestBuilder.setIndex(indexAliasName);
    }
    if(analyzer !=null)
    {
        analyzeRequestBuilder.setAnalyzer(analyzer);
    }
    
    if(tokenFilters !=null)
    {
        analyzeRequestBuilder.setTokenFilters(tokenFilters);
    }
    
    logger.debug("Analyze request is text: {}, analyzer: {}, tokenfilters: {}", new Object[]{analyzeRequestBuilder.request().text(), 
                                                                                analyzeRequestBuilder.request().analyzer(),
                                                                                analyzeRequestBuilder.request().tokenFilters()});
                                                                                        
    AnalyzeResponse analyzeResponse = analyzeRequestBuilder.get();
    
    try
    {
        if(analyzeResponse != null)
        {
            logger.debug("Analyze response is : {}", analyzeResponse.toXContent(jsonBuilder().startObject(), ToXContent.EMPTY_PARAMS).prettyPrint().string());
        }
    } catch (IOException e)
    {
        logger.error("Error printing response.", e);
    }
    
    for (AnalyzeToken analyzeToken : analyzeResponse.getTokens())
    {
        tokens.add(analyzeToken.getTerm());
    }
    return tokens;
}
开发者ID:jaibeermalik，项目名称:elasticsearch-tutorial，代码行数:45，代码来源:SetupIndexServiceImpl.java

示例15: convert

import org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse; //导入方法依赖的package包/类
@Override
public List<String> convert(final String text, final String... dummy) throws IOException {
    final AnalyzeResponse readingResponse =
            client.admin().indices().prepareAnalyze(analyzerSettings.getAnalyzerSettingsIndexName(), text)
                    .setAnalyzer(analyzerSettings.getReadingAnalyzerName(lang)).execute().actionGet();

    final AnalyzeResponse termResponse =
            client.admin().indices().prepareAnalyze(analyzerSettings.getAnalyzerSettingsIndexName(), text)
                    .setAnalyzer(analyzerSettings.getReadingTermAnalyzerName(lang)).execute().actionGet();

    final List<AnalyzeResponse.AnalyzeToken> readingTokenList = readingResponse.getTokens();
    final List<AnalyzeResponse.AnalyzeToken> termTokenList = termResponse.getTokens();

    final StringBuilder readingBuf = new StringBuilder(text.length());
    int offset = 0;
    for (int i = 0; i < readingTokenList.size(); i++) {
        final String term = termTokenList.get(i).getTerm();
        String reading = readingTokenList.get(i).getTerm();
        if (Strings.isNullOrEmpty(reading)) {
            reading = term;
        }
        reading = transliterator.transliterate(reading);

        final int pos = text.substring(offset).indexOf(term);
        if (pos > 0) {
            final String tmp = text.substring(offset, offset + pos);
            readingBuf.append(transliterator.transliterate(tmp));
            offset += pos;
        } else if (pos == -1) {
            continue;
        }

        readingBuf.append(reading);
        offset += term.length();
    }

    final List<String> list = new ArrayList<>(1);
    if (readingBuf.length() > 0) {
        list.add(readingBuf.toString());
    }
    return list;
}
开发者ID:codelibs，项目名称:fess-suggest，代码行数:43，代码来源:AnalyzerConverter.java


注：本文中的org.elasticsearch.action.admin.indices.analyze.AnalyzeResponse.getTokens方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。