Java StandardAnalyzer类代码示例

import org.apache.lucene.analysis.standard.StandardAnalyzer; //导入依赖的package包/类
private List<Post> findPosts() {
	try {
		FullTextSession fullTextSession = getFullTextSession((Session) entityManager.getDelegate());
		Builder builder = new Builder();
		String[] fields = new String[] { "message.text", "topic.subject" };
		MultiFieldQueryParser parser = new MultiFieldQueryParser(fields, new StandardAnalyzer());
		builder.add(parser.parse(POST_TEXT), MUST);
		builder.add(new TermQuery(new Term("topic.forum.id", "0")), MUST);
		builder.add(new TermQuery(new Term("topic.forum.category.id", "0")), MUST);
		builder.add(new WildcardQuery(new Term("poster.userId", "root")), MUST);
		addPostTimeQuery(builder);
		FullTextQuery fullTextQuery = fullTextSession.createFullTextQuery(builder.build(), Post.class);
		fullTextQuery.setSort(getSort());
		fullTextQuery.setFirstResult(0);
		fullTextQuery.setMaxResults(15);
		@SuppressWarnings("unchecked")
		List<Post> posts = fullTextQuery.list();
		return posts;
	} catch (ParseException e) {
		logger.severe("error");
		return null;
	}
}
 

import org.apache.lucene.analysis.standard.StandardAnalyzer; //导入依赖的package包/类
public void testMultiPhrasePrefixQuery() throws Exception {
    Analyzer analyzer = new StandardAnalyzer();
    Directory dir = newDirectory();
    String value = "The quick brown fox.";
    IndexReader ir = indexOneDoc(dir, "text", value, analyzer);
    MultiPhrasePrefixQuery query = new MultiPhrasePrefixQuery();
    query.add(new Term("text", "quick"));
    query.add(new Term("text", "brown"));
    query.add(new Term("text", "fo"));
    IndexSearcher searcher = newSearcher(ir);
    TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
    assertThat(topDocs.totalHits, equalTo(1));
    int docId = topDocs.scoreDocs[0].doc;
    CustomPassageFormatter passageFormatter = new CustomPassageFormatter("<b>", "</b>", new DefaultEncoder());
    CustomUnifiedHighlighter highlighter = new CustomUnifiedHighlighter(searcher, analyzer,
        passageFormatter, null, value, false);
    Snippet[] snippets = highlighter.highlightField("text", query, docId, 5);
    assertThat(snippets.length, equalTo(1));
    assertThat(snippets[0].getText(), equalTo("The <b>quick</b> <b>brown</b> <b>fox</b>."));
    ir.close();
    dir.close();
}
 

import org.apache.lucene.analysis.standard.StandardAnalyzer; //导入依赖的package包/类
public void testAllTermQuery() throws IOException {
    Directory dir = newDirectory();
    String value = "The quick brown fox.";
    Analyzer analyzer = new StandardAnalyzer();
    IndexReader ir = indexOneDoc(dir, "all", value, analyzer);
    AllTermQuery query = new AllTermQuery(new Term("all", "fox"));
    IndexSearcher searcher = newSearcher(ir);
    TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
    assertThat(topDocs.totalHits, equalTo(1));
    int docId = topDocs.scoreDocs[0].doc;
    CustomPassageFormatter passageFormatter = new CustomPassageFormatter("<b>", "</b>", new DefaultEncoder());
    CustomUnifiedHighlighter highlighter = new CustomUnifiedHighlighter(searcher, analyzer,
        passageFormatter, null, value, false);
    Snippet[] snippets = highlighter.highlightField("all", query, docId, 5);
    assertThat(snippets.length, equalTo(1));
    assertThat(snippets[0].getText(), equalTo("The quick brown <b>fox</b>."));
    ir.close();
    dir.close();
}
 

import org.apache.lucene.analysis.standard.StandardAnalyzer; //导入依赖的package包/类
public void testCommonTermsQuery() throws IOException {
    Directory dir = newDirectory();
    String value = "The quick brown fox.";
    Analyzer analyzer = new StandardAnalyzer();
    IndexReader ir = indexOneDoc(dir, "text", value, analyzer);
    CommonTermsQuery query = new CommonTermsQuery(BooleanClause.Occur.SHOULD, BooleanClause.Occur.SHOULD, 128);
    query.add(new Term("text", "quick"));
    query.add(new Term("text", "brown"));
    query.add(new Term("text", "fox"));
    IndexSearcher searcher = newSearcher(ir);
    TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
    assertThat(topDocs.totalHits, equalTo(1));
    int docId = topDocs.scoreDocs[0].doc;
    CustomPassageFormatter passageFormatter = new CustomPassageFormatter("<b>", "</b>", new DefaultEncoder());
    CustomUnifiedHighlighter highlighter = new CustomUnifiedHighlighter(searcher, analyzer,
        passageFormatter, null, value, false);
    Snippet[] snippets = highlighter.highlightField("text", query, docId, 5);
    assertThat(snippets.length, equalTo(1));
    assertThat(snippets[0].getText(), equalTo("The <b>quick</b> <b>brown</b> <b>fox</b>."));
    ir.close();
    dir.close();
}
 

import org.apache.lucene.analysis.standard.StandardAnalyzer; //导入依赖的package包/类
public void testAnalyzerAliasReferencesAlias() throws IOException {
    Settings settings = Settings.builder()
        .put("index.analysis.analyzer.foobar.alias","default")
        .put("index.analysis.analyzer.foobar.type", "german")
        .put("index.analysis.analyzer.foobar_search.alias","default_search")
        .put("index.analysis.analyzer.foobar_search.type", "default")
        .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
        // analyzer aliases are only allowed in 2.x indices
        .put(IndexMetaData.SETTING_VERSION_CREATED, VersionUtils.randomVersionBetween(random(), Version.V_2_0_0, Version.V_2_3_5))
        .build();
    AnalysisRegistry newRegistry = getNewRegistry(settings);
    IndexAnalyzers indexAnalyzers = getIndexAnalyzers(newRegistry, settings);

    assertThat(indexAnalyzers.get("default").analyzer(), is(instanceOf(GermanAnalyzer.class)));
    // analyzer types are bound early before we resolve aliases
    assertThat(indexAnalyzers.get("default_search").analyzer(), is(instanceOf(StandardAnalyzer.class)));
    assertWarnings("setting [index.analysis.analyzer.foobar.alias] is only allowed on index [test] because it was created before " +
            "5.x; analyzer aliases can no longer be created on new indices.",
            "setting [index.analysis.analyzer.foobar_search.alias] is only allowed on index [test] because it was created before " +
                    "5.x; analyzer aliases can no longer be created on new indices.");
}
 

import org.apache.lucene.analysis.standard.StandardAnalyzer; //导入依赖的package包/类
public void testVersionedAnalyzers() throws Exception {
    String yaml = "/org/elasticsearch/index/analysis/test1.yml";
    Settings settings2 = Settings.builder()
            .loadFromStream(yaml, getClass().getResourceAsStream(yaml))
            .put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toString())
            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.V_2_0_0)
            .build();
    AnalysisRegistry newRegistry = getNewRegistry(settings2);
    IndexAnalyzers indexAnalyzers = getIndexAnalyzers(newRegistry, settings2);

    // registry always has the current version
    assertThat(newRegistry.getAnalyzer("default"), is(instanceOf(NamedAnalyzer.class)));
    NamedAnalyzer defaultNamedAnalyzer = (NamedAnalyzer) newRegistry.getAnalyzer("default");
    assertThat(defaultNamedAnalyzer.analyzer(), is(instanceOf(StandardAnalyzer.class)));
    assertEquals(Version.CURRENT.luceneVersion, defaultNamedAnalyzer.analyzer().getVersion());

    // analysis service has the expected version
    assertThat(indexAnalyzers.get("standard").analyzer(), is(instanceOf(StandardAnalyzer.class)));
    assertEquals(Version.V_2_0_0.luceneVersion, indexAnalyzers.get("standard").analyzer().getVersion());
    assertEquals(Version.V_2_0_0.luceneVersion, indexAnalyzers.get("thai").analyzer().getVersion());

    assertThat(indexAnalyzers.get("custom7").analyzer(), is(instanceOf(StandardAnalyzer.class)));
    assertEquals(org.apache.lucene.util.Version.fromBits(3,6,0), indexAnalyzers.get("custom7").analyzer().getVersion());
}
 

import org.apache.lucene.analysis.standard.StandardAnalyzer; //导入依赖的package包/类
public void checkGeoQueryHighlighting(Query geoQuery) throws IOException, InvalidTokenOffsetsException {
    Map analysers = new HashMap<String, Analyzer>();
    analysers.put("text", new StandardAnalyzer());
    FieldNameAnalyzer fieldNameAnalyzer = new FieldNameAnalyzer(analysers);
    Query termQuery = new TermQuery(new Term("text", "failure"));
    Query boolQuery = new BooleanQuery.Builder().add(new BooleanClause(geoQuery, BooleanClause.Occur.SHOULD))
        .add(new BooleanClause(termQuery, BooleanClause.Occur.SHOULD)).build();
    org.apache.lucene.search.highlight.Highlighter highlighter =
        new org.apache.lucene.search.highlight.Highlighter(new CustomQueryScorer(boolQuery));
    String fragment = highlighter.getBestFragment(fieldNameAnalyzer.tokenStream("text", "Arbitrary text field which should not cause " +
        "a failure"), "Arbitrary text field which should not cause a failure");
    assertThat(fragment, equalTo("Arbitrary text field which should not cause a <B>failure</B>"));
    Query rewritten = boolQuery.rewrite(null);
    highlighter = new org.apache.lucene.search.highlight.Highlighter(new CustomQueryScorer(rewritten));
    fragment = highlighter.getBestFragment(fieldNameAnalyzer.tokenStream("text", "Arbitrary text field which should not cause " +
        "a failure"), "Arbitrary text field which should not cause a failure");
    assertThat(fragment, equalTo("Arbitrary text field which should not cause a <B>failure</B>"));
}
 

import org.apache.lucene.analysis.standard.StandardAnalyzer; //导入依赖的package包/类
public void testAnalyzeWildcard() {
    SimpleQueryParser.Settings settings = new SimpleQueryParser.Settings();
    settings.analyzeWildcard(true);
    Map<String, Float> weights = new HashMap<>();
    weights.put("field1", 1.0f);
    SimpleQueryParser parser = new MockSimpleQueryParser(new StandardAnalyzer(), weights, -1, settings);
    for (Operator op : Operator.values()) {
        BooleanClause.Occur defaultOp = op.toBooleanClauseOccur();
        parser.setDefaultOperator(defaultOp);
        Query query = parser.parse("first foo-bar-foobar* last");
        Query expectedQuery =
            new BooleanQuery.Builder()
                .add(new BooleanClause(new TermQuery(new Term("field1", "first")), defaultOp))
                .add(new BooleanQuery.Builder()
                    .add(new BooleanClause(new TermQuery(new Term("field1", "foo")), defaultOp))
                    .add(new BooleanClause(new TermQuery(new Term("field1", "bar")), defaultOp))
                    .add(new BooleanClause(new PrefixQuery(new Term("field1", "foobar")), defaultOp))
                    .build(), defaultOp)
                .add(new BooleanClause(new TermQuery(new Term("field1", "last")), defaultOp))
                .build();
        assertThat(query, equalTo(expectedQuery));
    }
}
 

import org.apache.lucene.analysis.standard.StandardAnalyzer; //导入依赖的package包/类
public void testNoParentNullFieldCreatedIfNoParentSpecified() throws Exception {
    Index index = new Index("_index", "testUUID");
    IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(index, Settings.EMPTY);
    NamedAnalyzer namedAnalyzer = new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer());
    IndexAnalyzers indexAnalyzers = new IndexAnalyzers(indexSettings, namedAnalyzer, namedAnalyzer, namedAnalyzer,
        Collections.emptyMap(), Collections.emptyMap());
    SimilarityService similarityService = new SimilarityService(indexSettings, Collections.emptyMap());
    MapperService mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry(), similarityService,
        new IndicesModule(emptyList()).getMapperRegistry(), () -> null);
    XContentBuilder mappingSource = jsonBuilder().startObject().startObject("some_type")
        .startObject("properties")
        .endObject()
        .endObject().endObject();
    mapperService.merge("some_type", new CompressedXContent(mappingSource.string()), MergeReason.MAPPING_UPDATE, false);
    Set<String> allFields = new HashSet<>(mapperService.simpleMatchToIndexNames("*"));
    assertTrue(allFields.contains("_parent"));
    assertFalse(allFields.contains("_parent#null"));
}
 

import org.apache.lucene.analysis.standard.StandardAnalyzer; //导入依赖的package包/类
public LuceneService() {
    try {
        this.keySerializer = new ObjectMapper();
        keySerializer.setVisibility(
                keySerializer.getSerializationConfig().
                        getDefaultVisibilityChecker().
                        withFieldVisibility(JsonAutoDetect.Visibility.ANY).
                        withGetterVisibility(JsonAutoDetect.Visibility.NONE).
                        withIsGetterVisibility(JsonAutoDetect.Visibility.NONE)
        );

        indexDirectory = new RAMDirectory();
        analyzer = new StandardAnalyzer(Version.LUCENE_4_9);

        IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_4_9, analyzer);
        iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
        indexWriter = new IndexWriter(indexDirectory, iwc);
        commit();
    } catch (IOException e) {
        throw new IndexingException(errorMessage(
                "could not create full-text index", 0), e);
    } catch (VirtualMachineError vme) {
        handleVirtualMachineError(vme);
    }
}
 

import org.apache.lucene.analysis.standard.StandardAnalyzer; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
	IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(INDEX_DIRECTORY)));
	IndexSearcher indexSearcher = new IndexSearcher(reader);

	Analyzer analyzer = new StandardAnalyzer();
	QueryParser queryParser = new QueryParser(FIELD_CONTENTS, analyzer);
	String searchString = "shakespeare";
	Query query = queryParser.parse(searchString);

	TopDocs results = indexSearcher.search(query, 5);
	ScoreDoc[] hits = results.scoreDocs;

	int numTotalHits = results.totalHits;
	System.out.println(numTotalHits + " total matching documents");

	for(int i=0;i<hits.length;++i) {
		int docId = hits[i].doc;
		Document d = indexSearcher.doc(docId);
		System.out.println((i + 1) + ". " + d.get("path") + " score=" + hits[i].score);
	}
}
 

import org.apache.lucene.analysis.standard.StandardAnalyzer; //导入依赖的package包/类
public SearchResult search(String index, String queryString, int page) {
    SearchResult searchResult = null;

    try {
        IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(Properties.getProperties().getProperty(Values.INDEX_LOCATION, Values.DEFAULT_INDEX_LOCATION))));
        IndexSearcher searcher = new IndexSearcher(reader);
        Analyzer analyzer = new StandardAnalyzer();

        // Search over the titles only for the moment
        QueryParser parser = new QueryParser(index, analyzer);
        Query query = parser.parse(queryString);

        searchResult = this.doPagingSearch(reader, searcher, query, queryString, page);
        reader.close();
    }
    catch(Exception ex) {}

    return searchResult;
}
 

import org.apache.lucene.analysis.standard.StandardAnalyzer; //导入依赖的package包/类
private void synTokenQuery(String search, final int numbOfResults, final double minLuceneScore,
		Map<String, Float> result, IndexSearcher searcher) throws ParseException, IOException {

	QueryParser parser = new QueryParser(Version.LUCENE_46, "surfaceFormTokens",
			new StandardAnalyzer(Version.LUCENE_46));

	search = QueryParser.escape(search);

	Query q = parser.parse(search);
	/*
	 * Works only in String field!!
	 */
	// Query q = new FuzzyQuery(new Term("surfaceFormTokens",
	// QueryParser.escape(search)), 2);

	TopDocs top = searcher.search(q, numbOfResults);

	for (ScoreDoc doc : top.scoreDocs) {
		if (doc.score >= minLuceneScore) {
			final String key = searcher.doc(doc.doc).get("conceptID");
			if (result.getOrDefault(key, 0f) < doc.score) {
				result.put(key, doc.score);
			}
		}
	}
}
 

import org.apache.lucene.analysis.standard.StandardAnalyzer; //导入依赖的package包/类
/**
 * Deletes features from specified feature files from project's index
 *
 * @param projectId a project to delete index entries
 * @param fileIds a list of Pair of feature types to file Ids, which entries to delete. To delete gene file
 *                entries, pass FeatureType.GENE
 */
public void deleteFromIndexByFileId(final long projectId, List<Pair<FeatureType, Long>> fileIds) {
    if (fileIds == null || fileIds.isEmpty() || !fileManager.indexForProjectExists(projectId)) {
        return;
    }

    try (
        StandardAnalyzer analyzer = new StandardAnalyzer();
        Directory index = fileManager.getIndexForProject(projectId);
        IndexWriter writer = new IndexWriter(index, new IndexWriterConfig(analyzer).setOpenMode(
                                                                    IndexWriterConfig.OpenMode.CREATE_OR_APPEND))
    ) {
        if (fileManager.indexForProjectExists(projectId)) {
            for (Pair<FeatureType, Long> id : fileIds) {
                deleteDocumentByTypeAndId(id.getKey(), id.getValue(), writer);
            }
        }
    } catch (IOException e) {
        LOGGER.error("Exception while deleting from index:", e);
    }
}
 

import org.apache.lucene.analysis.standard.StandardAnalyzer; //导入依赖的package包/类
public FbEntitySearcher(String indexDir, int numOfDocs, String searchingStrategy) throws IOException {

    LogInfo.begin_track("Constructing Searcher");
    if (!searchingStrategy.equals("exact") && !searchingStrategy.equals("inexact"))
      throw new RuntimeException("Bad searching strategy: " + searchingStrategy);
    this.searchStrategy = searchingStrategy;

    queryParser = new QueryParser(
        Version.LUCENE_44,
        FbIndexField.TEXT.fieldName(),
        searchingStrategy.equals("exact") ? new KeywordAnalyzer() : new StandardAnalyzer(Version.LUCENE_44));
    LogInfo.log("Opening index dir: " + indexDir);
    IndexReader indexReader = DirectoryReader.open(SimpleFSDirectory.open(new File(indexDir)));
    indexSearcher = new IndexSearcher(indexReader);
    LogInfo.log("Opened index with " + indexReader.numDocs() + " documents.");

    this.numOfDocs = numOfDocs;
    LogInfo.end_track();
  }