C# StandardAnalyzer.TokenStream方法代码示例

        public void v()
        {

            //Analyzer analyzer = new CJKAnalyzer();
            //TokenStream tokenStream = analyzer.TokenStream("", new StringReader("我爱你中国China中华人名共和国"));
            //Lucene.Net.Analysis.Token token = null;
            //while ((token = tokenStream.Next()) != null)
            //{
            //    Response.Write(token.TermText() + "<br/>");
            //}

            Lucene.Net.Analysis.Standard.StandardAnalyzer a = new Lucene.Net.Analysis.Standard.StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30);
            string s = "我日中华人民共和国";
            System.IO.StringReader reader = new System.IO.StringReader(s);
            Lucene.Net.Analysis.TokenStream ts = a.TokenStream(s, reader);
            bool hasnext = ts.IncrementToken();
            Lucene.Net.Analysis.Tokenattributes.ITermAttribute ita;
            while (hasnext)
            {
                ita = ts.GetAttribute<Lucene.Net.Analysis.Tokenattributes.ITermAttribute>();
                Console.WriteLine(ita.Term);
                hasnext = ts.IncrementToken();
            }
            ts.CloneAttributes();
            reader.Close();
            a.Close();
            Console.ReadKey();
        }

        public List<IndexResult> Search(string terms)
        {
            List<IndexResult> retObj = new List<IndexResult>();
            var analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30);

            using (var searcher = new IndexSearcher(FSDirectory.Open(IndexDirectory)))
            {

                // parse the query, "text" is the default field to search
                var parser = new MultiFieldQueryParser(Lucene.Net.Util.Version.LUCENE_30, new[] { "text", "title", "urlkey", "searchterms" }, analyzer);

                Query query = parser.Parse(terms);

                TopDocs hits = searcher.Search(query, 200);

                SimpleFragmenter fragmenter = new SimpleFragmenter(80);
                QueryScorer scorer = new QueryScorer(query);
                Highlighter highlighter = new Highlighter(scorer);
                highlighter.TextFragmenter = fragmenter;

                for (int i = 0; i < hits.TotalHits; i++)
                {
                    // get the document from index
                    Document doc = searcher.Doc(hits.ScoreDocs[i].Doc);

                    TokenStream stream = analyzer.TokenStream("", new StringReader(doc.Get("text")));

                    String sample = highlighter.GetBestFragments(stream, doc.Get("text"), 2, "...");
                    String title = doc.Get("title");
                    String urlkey = doc.Get("urlkey");
                    String type = doc.Get("type");

                    retObj.Add(new IndexResult()
                    {
                        Sample = sample,
                        Title = title,
                        Type = type,
                        UrlKey = urlkey

                    });

                }

                return retObj;
            }
        }

        public void TestMethod1()
        {
            Lucene.Net.Analysis.Standard.StandardAnalyzer a = new Lucene.Net.Analysis.Standard.StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30);
            string s = "我日中华人民共和国";
            System.IO.StringReader reader = new System.IO.StringReader(s);
            Lucene.Net.Analysis.TokenStream ts = a.TokenStream(s, reader);
            bool hasnext = ts.IncrementToken();
            Lucene.Net.Analysis.Tokenattributes.ITermAttribute ita;

            while (hasnext)
            {
                ita = ts.GetAttribute<Lucene.Net.Analysis.Tokenattributes.ITermAttribute>();
                Console.WriteLine(ita.Term);
                hasnext = ts.IncrementToken();
            }

            Console.WriteLine("over");


            ts.CloneAttributes();
            reader.Close();
            a.Close();

        }

 void ViewTokenTerms()
 {
     var analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_29);
     var value = new System.IO.StringReader("Description2 1");
     var tokenStream = analyzer.TokenStream(FieldKeys.Detail_Desc, value);
     OffsetAttribute offSet = tokenStream.GetAttribute(typeof(OffsetAttribute)) as OffsetAttribute;
     TermAttribute termAttr = tokenStream.GetAttribute(typeof(TermAttribute)) as TermAttribute;
     while (tokenStream.IncrementToken())
     {
         offSet.StartOffset();
         offSet.EndOffset();
         var term = termAttr.Term();
     }
 }

        public static DataTable searchPitanja(string pretraga)
        {
            DataTable ResultsPitanja = new DataTable();
            // create the searcher
            // index is placed in "index" subdirectory
            string indexDirectory = "J:/Triglav_Web_App/Triglav/Web/Lucene/Pitanja";
            var analyzer = new StandardAnalyzer(Version.LUCENE_30);
            IndexSearcher searcher = new IndexSearcher(FSDirectory.Open(indexDirectory));

            // parse the query, "text" is the default field to search
            var parser = new MultiFieldQueryParser(Version.LUCENE_30, new[] { "Naslov", "Sadrzaj", "Tagovi" }, analyzer);
            //var parser = new QueryParser(Version.LUCENE_30, "Sadrzaj", analyzer);
            Query query = parser.Parse(pretraga);

            //// create the result DataTable
            ResultsPitanja.Columns.Add("id", typeof(Int32));
            ResultsPitanja.Columns.Add("Naslov", typeof(string));
            ResultsPitanja.Columns.Add("Sadrzaj", typeof(string));
            ResultsPitanja.Columns.Add("Tagovi", typeof(string));
            ResultsPitanja.Columns.Add("DatumKreiranja", typeof(DateTime));
            ResultsPitanja.Columns.Add("DatumZadnjeIzmjene", typeof(DateTime));
            ResultsPitanja.Columns.Add("DatumZadnjeAktivnosti", typeof(DateTime));
            ResultsPitanja.Columns.Add("DatumZatvaranjaPosta", typeof(DateTime));
            ResultsPitanja.Columns.Add("PrihvaceniOdgovori", typeof(Int32));
            ResultsPitanja.Columns.Add("BrojOdgovora", typeof(Int32));
            ResultsPitanja.Columns.Add("BrojKomentara", typeof(Int32));
            ResultsPitanja.Columns.Add("BrojOmiljenih", typeof(Int32));
            ResultsPitanja.Columns.Add("BrojPregleda", typeof(Int32));
            ResultsPitanja.Columns.Add("BrojPoena", typeof(Int32));
            ResultsPitanja.Columns.Add("VlasnikID", typeof(Int32));
            ResultsPitanja.Columns.Add("VlasnikNadimak", typeof(string));
            ResultsPitanja.Columns.Add("PromijenioID", typeof(Int32));
            ResultsPitanja.Columns.Add("RoditeljskiPostID", typeof(Int32));
            //Results.Columns.Add("PodKategorija", typeof(Int32));
            ResultsPitanja.Columns.Add("PostVrsta", typeof(Int32));
            // ResultsPitanja.Columns.Add("SlikaURL", typeof(string));
            ResultsPitanja.Columns.Add("temp", typeof(string));
            ResultsPitanja.Columns.Add("Likes", typeof(Int32));
            ResultsPitanja.Columns.Add("Unlikes", typeof(Int32));
            ResultsPitanja.Columns.Add("Sazetak", typeof(string));
            ResultsPitanja.Columns.Add("BrojRangiranja", typeof(Int32));
            ResultsPitanja.Columns.Add("PrihvacenaIzmjena", typeof(Int32));
            ResultsPitanja.Columns.Add("Podnaslov", typeof(string));
            ResultsPitanja.Columns.Add("Broj.Razgovora", typeof(Int32));
            ResultsPitanja.Columns.Add("sample", typeof(string));

            // search
            TopDocs hits = searcher.Search(query, 5);

            //E this.total = hits.TotalHits;

            // create highlighter
            IFormatter formatter = new SimpleHTMLFormatter("<span style=\"font-weight:bold; background-color: #e5ecf9; \">", "</span>");
            SimpleFragmenter fragmenter = new SimpleFragmenter(80);
            QueryScorer scorer = new QueryScorer(query);
            Highlighter highlighter = new Highlighter(formatter, scorer);
            highlighter.TextFragmenter = fragmenter;

            for (int i = 0; i < hits.ScoreDocs.Count(); i++)
            {
                // get the document from index
                Document doc = searcher.Doc(hits.ScoreDocs[i].Doc);

                TokenStream stream = analyzer.TokenStream("", new StringReader(doc.Get("Sadrzaj")));
                String sample = highlighter.GetBestFragments(stream, doc.Get("Sadrzaj"), 3, "...");

                //String path = doc.Get("path");

                // create a new row with the result data
                DataRow rowPitanja = ResultsPitanja.NewRow();

                rowPitanja["id"] = doc.Get("id");
                rowPitanja["Naslov"] = doc.Get("Naslov");
                rowPitanja["Sadrzaj"] = sample; //doc.Get("Sadrzaj");
                rowPitanja["Tagovi"] = doc.Get("Tagovi");
                rowPitanja["DatumKreiranja"] = doc.Get("DatumKreiranja");
                rowPitanja["DatumZadnjeIzmjene"] = doc.Get("DatumZadnjeIzmjene");
                rowPitanja["DatumZadnjeAktivnosti"] = doc.Get("DatumZadnjeAktivnosti");
                //row["DatumZatvaranjaPosta"] = doc.Get("DatumZatvaranjaPosta");
                rowPitanja["PrihvaceniOdgovori"] = doc.Get("PrihvaceniOdgovori");
                rowPitanja["BrojOdgovora"] = doc.Get("BrojOdgovora");
                rowPitanja["BrojKomentara"] = doc.Get("BrojKomentara");
                rowPitanja["BrojOmiljenih"] = doc.Get("BrojOmiljenih");
                rowPitanja["BrojPregleda"] = doc.Get("BrojPregleda");
                rowPitanja["BrojPoena"] = doc.Get("BrojPoena");
                //row["VlasnikID"] = doc.Get("VlasnikID");
                rowPitanja["VlasnikNadimak"] = doc.Get("VlasnikNadimak");
                //row["PromijenioID"] = doc.Get("PromijenioID");
                //row["RoditeljskiPostID"] = doc.Get("RoditeljskiPostID");
                //row["PodKategorija"] = doc.Get("PodKategorija");
                rowPitanja["PostVrsta"] = doc.Get("PostVrsta");
                //rowPitanja["SlikaURL"] = doc.Get("SlikaURL");
                //row["temp"] = doc.Get("temp");
                rowPitanja["Likes"] = doc.Get("Likes");
                rowPitanja["Unlikes"] = doc.Get("Unlikes");
                rowPitanja["Sazetak"] = doc.Get("Sazetak");
                rowPitanja["BrojRangiranja"] = doc.Get("BrojRangiranja");
                rowPitanja["PrihvacenaIzmjena"] = doc.Get("PrihvacenaIzmjena");
                rowPitanja["Podnaslov"] = doc.Get("Podnaslov");
                //row["Broj.Razgovora"] = doc.Get("Broj.Razgovora");
//.........这里部分代码省略.........

//.........这里部分代码省略.........
            //}

            //foreach (string project in Projects.Distinct())
            //{
            //    TermQuery termq1 = new TermQuery(new Term("path", project));
            //    bquery.Add(termq1, Occur.MUST);
            //    FuzzyQuery termq = new FuzzyQuery(new Term("path", project), 0.5f, 0);
            //    bquery.Add(termq, Occur.MUST);
            //}

            //bquery.Add(new TermQuery(new Term("Project", "DEV")), Occur.SHOULD);

            //List<ScoreDoc> TempArrList = new List<ScoreDoc>();

            TopDocs hits = searcher.Search(bquery, null, 10000);

            //TopDocs hits = new TopDocs(TempArrList.Count(), TempArrList.ToArray(), hitsWithText.MaxScore);
            //hits.ScoreDocs.CopyTo(hits.ScoreDocs, 0);
            //hits.ScoreDocs = hits.ScoreDocs.OrderBy(obj => searcher.Doc(obj.Doc).Get("path")).ToArray();

            if (Projects.Count() != 0)
            {
                hits.ScoreDocs = hits.ScoreDocs.Where(obj => Projects.Contains(Path.GetDirectoryName(searcher.Doc(obj.Doc).Get("path")))).Distinct().ToArray();
            }

            //foreach (string project in Projects.Distinct())
            //{
            //    //hits.ScoreDocs = hits.ScoreDocs.Where(obj => Regex.IsMatch(searcher.Doc(obj.Doc).Get("path").Replace(@"\", @"\\"), @".*" + project.Replace(@"\", @"\\") + ".*")).ToArray();
            //    string s = Path.GetDirectoryName("\\SAGITEC-1629\\Soogle\\CARS\\bhagyashree.txt");
            //    hits.ScoreDocs = hits.ScoreDocs.Where(obj => Path.GetDirectoryName(searcher.Doc(obj.Doc).Get("path")).Contains(project)).ToArray();
            //}

            this.total = hits.ScoreDocs.Count();

            this.startAt = InitStartAt();

            int resultsCount = Math.Min(total, this.maxResults + this.startAt);

            // create highlighter
            IFormatter formatter = new SimpleHTMLFormatter("<span style=\"font-weight:bold;background-color:yellow;\">", "</span>");
            SimpleFragmenter fragmenter = new SimpleFragmenter(200);
            QueryScorer scorer = new QueryScorer(bquery);
            Highlighter highlighter = new Highlighter(formatter, scorer);
            highlighter.TextFragmenter = fragmenter;

            int j = 0;

            for (int i = startAt; i < resultsCount; i++)
            {
                Document doc = searcher.Doc(hits.ScoreDocs[i].Doc);
                String path = doc.Get("path");
                string getExtension = doc.Get("Extension");

                TokenStream stream = analyzer.TokenStream("", new StringReader(doc.Get("text")));
                String sample = "";
                try
                {
                    string document = doc.Get("text");
                    if (getExtension.ToLower() == ".png" || getExtension.ToLower() == ".jpg" || getExtension.ToLower() == ".gif" || getExtension.ToLower() == ".bmp")
                    {
                        sample = "";
                    }
                    else
                    {
                        sample = highlighter.GetBestFragment(stream, document);//, 2, "...");
                    }

                }
                catch (Exception ex)
                {
                }

                // create a new row with the result data
                DataRow row = this.Results.NewRow();
                row["title"] = doc.Get("title");
                row["path"] = "http://sagitec-1629/KNBASE/" + path.Replace(@"\", "/").Replace("//SAGITEC-1629/Soogle/", "");
                row["url"] = "http://sagitec-1629/KNBASE/" + path.Replace(@"\", "/").Replace("//SAGITEC-1629/Soogle/", "");
                row["sample"] = sample;
                if (path.Contains('.'))
                {
                    row["Type"] = GetMIMEType(path);
                }
                //if (!Projects.Contains(doc.Get("Project")) || !allType.Contains(doc.Get("EXTPRP")))
                //{
                this.Results.Rows.Add(row);
                //}
                j++;

            }

            Repeater1.DataSource = Results;
            Repeater1.DataBind();

            searcher.Dispose();

            // result information
            this.duration = DateTime.Now - start;
            this.fromItem = startAt + 1;
            this.toItem = Math.Min(startAt + maxResults, total);
        }

//.........这里部分代码省略.........
                //        {
                //            string outp = highlighter.GetBestFragment(stream, document);
                //            if (outp != null)
                //                sample = ReplaceSpecialChar(outp.Trim()); //, 2, "...");
                //            else
                //                sample = Limit(doc.Get("text").Trim(), 200);
                //        }

                //    }
                //    catch (Exception ex)
                //    {
                //    }

                //    // create a new row with the result data
                //    DataRow row = this.Results.NewRow();
                //    row["title"] = doc.Get("title");
                //    row["path"] = ApplicationPath + path.Replace(@"\", "/").Replace(VirtualPath, "");
                //    row["url"] = ApplicationPath + path.Replace(@"\", "/").Replace(VirtualPath, "");
                //    row["sample"] = sample;
                //    if (path.Contains('.'))
                //    {
                //        row["Type"] = GetMIMEType(path);
                //    }

                //    this.Results.Rows.Add(row);
                //}

                for (int i = 0; i < this.total; i++)
                {
                    Document doc = searcher.Doc(hits.ScoreDocs[i].Doc);
                    String path = doc.Get("path");
                    string getExtension = doc.Get("Extension");

                    TokenStream stream = analyzer.TokenStream("", new StringReader(doc.Get("text")));
                    String sample = "";
                    try
                    {
                        string document = doc.Get("text");
                        if (getExtension.ToLower() == ".png" || getExtension.ToLower() == ".jpg" || getExtension.ToLower() == ".gif" || getExtension.ToLower() == ".bmp" || getExtension.ToLower() == ".jpeg")
                        {
                            sample = "";
                        }
                        else
                        {
                            string outp = highlighter.GetBestFragment(stream, document);
                            if (outp != null)
                                sample = Limit(outp.Trim(), 200); //, 2, "...");
                            else
                                sample = Limit(doc.Get("text").Trim(), 200);
                        }

                    }
                    catch (Exception ex)
                    {
                    }

                    // create a new row with the result data
                    DataRow row = Results.NewRow();
                    row["title"] = doc.Get("title");
                    row["path"] = ApplicationPath + path.Replace(@"\", "/").Replace(VirtualPath, "");
                    row["url"] = ApplicationPath + path.Replace(@"\", "/").Replace(VirtualPath, "");
                    row["sample"] = sample;
                    if (path.Contains('.'))
                    {
                        row["Type"] = GetMIMEType(path);
                    }

        public void TestUnRewrittenQuery()
        {
            var helper = new TestHighlightRunner();
            helper.TestAction = () =>
                                    {
                                        numHighlights = 0;
                                        // test to show how rewritten query can still be used
                                        searcher = new IndexSearcher(ramDir, true);
                                        Analyzer analyzer = new StandardAnalyzer(TEST_VERSION);

                                        QueryParser parser = new QueryParser(TEST_VERSION, FIELD_NAME, analyzer);
                                        Query query = parser.Parse("JF? or Kenned*");
                                        Console.WriteLine("Searching with primitive query");
                                        // forget to set this and...
                                        // query=query.Rewrite(reader);
                                        TopDocs hits = searcher.Search(query, null, 1000);

                                        // create an instance of the highlighter with the tags used to surround
                                        // highlighted text
                                        // QueryHighlightExtractor highlighter = new
                                        // QueryHighlightExtractor(this,
                                        // query, new StandardAnalyzer(TEST_VERSION));

                                        int maxNumFragmentsRequired = 3;

                                        for (int i = 0; i < hits.TotalHits; i++)
                                        {
                                            String text = searcher.Doc(hits.ScoreDocs[i].Doc).Get(FIELD_NAME);
                                            TokenStream tokenStream = analyzer.TokenStream(FIELD_NAME,
                                                                                           new StringReader(text));
                                            Highlighter highlighter = helper.GetHighlighter(query, FIELD_NAME,
                                                                                            tokenStream,
                                                                                            this, false);

                                            highlighter.TextFragmenter = new SimpleFragmenter(40);

                                            String highlightedText = highlighter.GetBestFragments(tokenStream, text,
                                                                                                  maxNumFragmentsRequired,
                                                                                                  "...");

                                            Console.WriteLine(highlightedText);
                                        }
                                        // We expect to have zero highlights if the query is multi-terms and is
                                        // not
                                        // rewritten!
                                        Assert.IsTrue(numHighlights == 0,
                                                      "Failed to find correct number of highlights " + numHighlights +
                                                      " found");
                                    };

            helper.Start();
        }