当前位置: 首页>>代码示例>>C#>>正文


C# Token.TermBuffer方法代码示例

本文整理汇总了C#中Lucene.Net.Analysis.Token.TermBuffer方法的典型用法代码示例。如果您正苦于以下问题:C# Token.TermBuffer方法的具体用法?C# Token.TermBuffer怎么用?C# Token.TermBuffer使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在Lucene.Net.Analysis.Token的用法示例。


在下文中一共展示了Token.TermBuffer方法的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的C#代码示例。

示例1: TestCtor

 public virtual void  TestCtor()
 {
     Token t = new Token();
     char[] content = "hello".ToCharArray();
     t.SetTermBuffer(content, 0, content.Length);
     char[] buf = t.TermBuffer();
     Assert.AreNotEqual(t.TermBuffer(), content);
     Assert.AreEqual("hello", t.Term);
     Assert.AreEqual("word", t.Type);
     Assert.AreEqual(0, t.Flags);
     
     t = new Token(6, 22);
     t.SetTermBuffer(content, 0, content.Length);
     Assert.AreEqual("hello", t.Term);
     Assert.AreEqual("(hello,6,22)", t.ToString());
     Assert.AreEqual("word", t.Type);
     Assert.AreEqual(0, t.Flags);
     
     t = new Token(6, 22, 7);
     t.SetTermBuffer(content, 0, content.Length);
     Assert.AreEqual("hello", t.Term);
     Assert.AreEqual("(hello,6,22)", t.ToString());
     Assert.AreEqual(7, t.Flags);
     
     t = new Token(6, 22, "junk");
     t.SetTermBuffer(content, 0, content.Length);
     Assert.AreEqual("hello", t.Term);
     Assert.AreEqual("(hello,6,22,type=junk)", t.ToString());
     Assert.AreEqual(0, t.Flags);
 }
开发者ID:Nangal,项目名称:lucene.net,代码行数:30,代码来源:TestToken.cs

示例2: TestMixedStringArray

		public virtual void  TestMixedStringArray()
		{
			Token t = new Token("hello", 0, 5);
			Assert.AreEqual(t.TermText(), "hello");
			Assert.AreEqual(t.TermLength(), 5);
			Assert.AreEqual(new System.String(t.TermBuffer(), 0, 5), "hello");
			t.SetTermText("hello2");
			Assert.AreEqual(t.TermLength(), 6);
			Assert.AreEqual(new System.String(t.TermBuffer(), 0, 6), "hello2");
			t.SetTermBuffer("hello3".ToCharArray(), 0, 6);
			Assert.AreEqual(t.TermText(), "hello3");
			
			// Make sure if we get the buffer and change a character
			// that termText() reflects the change
			char[] buffer = t.TermBuffer();
			buffer[1] = 'o';
			Assert.AreEqual(t.TermText(), "hollo3");
		}
开发者ID:vikasraz,项目名称:indexsearchutils,代码行数:18,代码来源:TestToken.cs

示例3: AddToken

 void AddToken(Token oriToken, int termBufferOffset, int termBufferLength, byte type)
 {
     Token token = new Token(oriToken.TermBuffer(), termBufferOffset, termBufferLength,
         oriToken.StartOffset + termBufferOffset, oriToken.StartOffset + termBufferOffset + termBufferLength);
     if (type == (byte)UnicodeCategory.DecimalDigitNumber)
         token.Type = Word.TYPE_DIGIT;
     else
         token.Type = Word.TYPE_LETTER;
     tokenQueue.Enqueue(token);
 }
开发者ID:vebin,项目名称:Lucene.Net.Analysis.MMSeg,代码行数:10,代码来源:CutLeterDigitFilter.cs

示例4: TestResize

 public virtual void  TestResize()
 {
     Token t = new Token();
     char[] content = "hello".ToCharArray();
     t.SetTermBuffer(content, 0, content.Length);
     for (int i = 0; i < 2000; i++)
     {
         t.ResizeTermBuffer(i);
         Assert.IsTrue(i <= t.TermBuffer().Length);
         Assert.AreEqual("hello", t.Term);
     }
 }
开发者ID:Nangal,项目名称:lucene.net,代码行数:12,代码来源:TestToken.cs

示例5: Next

 public override Token Next(Token result)
 {
     result = input.Next(result);
     if (result != null)
     {
         if (stemmer.Stem(result.TermBuffer(), 0, result.termLength))
             result.SetTermBuffer(stemmer.GetResultBuffer(), 0, stemmer.GetResultLength());
         return result;
     }
     else
         return null;
 }
开发者ID:vikasraz,项目名称:indexsearchutils,代码行数:12,代码来源:PorterStemFilter.cs

示例6: Next

        public override Token Next(/* in */ Token reusableToken)
        {
            System.Diagnostics.Debug.Assert(reusableToken != null);
            reusableToken.Clear();
            int length = 0;
            int start = bufferIndex;
            char[] buffer = reusableToken.TermBuffer();
            while (true)
            {

                if (bufferIndex >= dataLen)
                {
                    offset += dataLen;
                    dataLen = input is Lucene.Net.Index.ReusableStringReader ? ((Lucene.Net.Index.ReusableStringReader) input).Read(ioBuffer) : input.Read((System.Char[]) ioBuffer, 0, ioBuffer.Length);
                    if (dataLen <= 0)
                    {
                        if (length > 0)
                            break;
                        else
                            return null;
                    }
                    bufferIndex = 0;
                }

                char c = ioBuffer[bufferIndex++];

                if (IsTokenChar(c))
                {
                    // if it's a token char

                    if (length == 0)
                        // start of token
                        start = offset + bufferIndex - 1;
                    else if (length == buffer.Length)
                        buffer = reusableToken.ResizeTermBuffer(1 + length);

                    buffer[length++] = Normalize(c); // buffer it, normalized

                    if (length == MAX_WORD_LEN)
                        // buffer overflow!
                        break;
                }
                else if (length > 0)
                    // at non-Letter w/ chars
                    break; // return 'em
            }

            reusableToken.SetTermLength(length);
            reusableToken.SetStartOffset(start);
            reusableToken.SetEndOffset(start + length);
            return reusableToken;
        }
开发者ID:cqm0609,项目名称:lucene-file-finder,代码行数:52,代码来源:CharTokenizer.cs

示例7: Next

        public override Token Next(Token token)
        {
            token.Clear();
            int length = 0;
            int start = bufferIndex;
            char[] buffer = token.TermBuffer();
            while (true)
            {

                if (bufferIndex >= dataLen)
                {
                    offset += dataLen;
                    dataLen = input is Lucene.Net.Index.DocumentsWriter.ReusableStringReader ? ((Lucene.Net.Index.DocumentsWriter.ReusableStringReader) input).Read(ioBuffer) : input.Read((System.Char[]) ioBuffer, 0, ioBuffer.Length);
                    if (dataLen <= 0)
                    {
                        if (length > 0)
                            break;
                        else
                            return null;
                    }
                    bufferIndex = 0;
                }

                char c = ioBuffer[bufferIndex++];

                if (IsTokenChar(c))
                {
                    // if it's a token char

                    if (length == 0)
                        // start of token
                        start = offset + bufferIndex - 1;
                    else if (length == buffer.Length)
                        buffer = token.ResizeTermBuffer(1 + length);

                    buffer[length++] = Normalize(c); // buffer it, normalized

                    if (length == MAX_WORD_LEN)
                        // buffer overflow!
                        break;
                }
                else if (length > 0)
                    // at non-Letter w/ chars
                    break; // return 'em
            }

            token.termLength = length;
            token.startOffset = start;
            token.endOffset = start + length;
            return token;
        }
开发者ID:vikasraz,项目名称:indexsearchutils,代码行数:51,代码来源:CharTokenizer.cs

示例8: Next

        public override Token Next(Token result)
        {
            result = input.Next(result);
            if (result != null)
            {

                char[] buffer = result.TermBuffer();
                int length = result.termLength;
                for (int i = 0; i < length; i++)
                    buffer[i] = System.Char.ToLower(buffer[i]);

                return result;
            }
            else
                return null;
        }
开发者ID:vikasraz,项目名称:indexsearchutils,代码行数:16,代码来源:LowerCaseFilter.cs

示例9: Next

 public override Token Next(Token result)
 {
     if (!done)
     {
         done = true;
         int upto = 0;
         result.Clear();
         char[] buffer = result.TermBuffer();
         while (true)
         {
             int length = input.Read(buffer, upto, buffer.Length - upto);
             if (length <= 0)
                 break;
             upto += length;
             if (upto == buffer.Length)
                 buffer = result.ResizeTermBuffer(1 + buffer.Length);
         }
         result.termLength = upto;
         return result;
     }
     return null;
 }
开发者ID:vikasraz,项目名称:indexsearchutils,代码行数:22,代码来源:KeywordTokenizer.cs

示例10: Next

 public override Token Next(/* in */ Token reusableToken)
 {
     System.Diagnostics.Debug.Assert(reusableToken != null);
     if (!done)
     {
         done = true;
         int upto = 0;
         reusableToken.Clear();
         char[] buffer = reusableToken.TermBuffer();
         while (true)
         {
             int length = input.Read(buffer, upto, buffer.Length - upto);
             if (length <= 0)
                 break;
             upto += length;
             if (upto == buffer.Length)
                 buffer = reusableToken.ResizeTermBuffer(1 + buffer.Length);
         }
         reusableToken.SetTermLength(upto);
         return reusableToken;
     }
     return null;
 }
开发者ID:cqm0609,项目名称:lucene-file-finder,代码行数:23,代码来源:KeywordTokenizer.cs

示例11: Next

        public override Token Next(Token result)
		{
            result = input.Next(result);
            if (result != null)
            {
                char[] buffer = result.TermBuffer();
                int length = result.TermLength();
                // If no characters actually require rewriting then we
                // just return token as-is:
                for (int i = 0; i < length; i++)
                {
                    char c = buffer[i];
                    if (c >= '\u00c0' && c <= '\u0178')
                    {
                        RemoveAccents(buffer, length);
                        result.SetTermBuffer(output, 0, outputPos);
                        break;
                    }
                }
                return result;
            }
            else
                return null;
        }
开发者ID:vikasraz,项目名称:indexsearchutils,代码行数:24,代码来源:ISOLatin1AccentFilter.cs

示例12: TestGrow

 public virtual void  TestGrow()
 {
     Token t = new Token();
     System.Text.StringBuilder buf = new System.Text.StringBuilder("ab");
     for (int i = 0; i < 20; i++)
     {
         char[] content = buf.ToString().ToCharArray();
         t.SetTermBuffer(content, 0, content.Length);
         Assert.AreEqual(buf.Length, t.TermLength());
         Assert.AreEqual(buf.ToString(), t.Term);
         buf.Append(buf.ToString());
     }
     Assert.AreEqual(1048576, t.TermLength());
     Assert.AreEqual(1179654, t.TermBuffer().Length);
     
     // now as a string, first variant
     t = new Token();
     buf = new System.Text.StringBuilder("ab");
     for (int i = 0; i < 20; i++)
     {
         System.String content = buf.ToString();
         t.SetTermBuffer(content, 0, content.Length);
         Assert.AreEqual(content.Length, t.TermLength());
         Assert.AreEqual(content, t.Term);
         buf.Append(content);
     }
     Assert.AreEqual(1048576, t.TermLength());
     Assert.AreEqual(1179654, t.TermBuffer().Length);
     
     // now as a string, second variant
     t = new Token();
     buf = new System.Text.StringBuilder("ab");
     for (int i = 0; i < 20; i++)
     {
         System.String content = buf.ToString();
         t.SetTermBuffer(content);
         Assert.AreEqual(content.Length, t.TermLength());
         Assert.AreEqual(content, t.Term);
         buf.Append(content);
     }
     Assert.AreEqual(1048576, t.TermLength());
     Assert.AreEqual(1179654, t.TermBuffer().Length);
     
     // Test for slow growth to a long term
     t = new Token();
     buf = new System.Text.StringBuilder("a");
     for (int i = 0; i < 20000; i++)
     {
         System.String content = buf.ToString();
         t.SetTermBuffer(content);
         Assert.AreEqual(content.Length, t.TermLength());
         Assert.AreEqual(content, t.Term);
         buf.Append("a");
     }
     Assert.AreEqual(20000, t.TermLength());
     Assert.AreEqual(20167, t.TermBuffer().Length);
     
     // Test for slow growth to a long term
     t = new Token();
     buf = new System.Text.StringBuilder("a");
     for (int i = 0; i < 20000; i++)
     {
         System.String content = buf.ToString();
         t.SetTermBuffer(content);
         Assert.AreEqual(content.Length, t.TermLength());
         Assert.AreEqual(content, t.Term);
         buf.Append("a");
     }
     Assert.AreEqual(20000, t.TermLength());
     Assert.AreEqual(20167, t.TermBuffer().Length);
 }
开发者ID:Nangal,项目名称:lucene.net,代码行数:71,代码来源:TestToken.cs

示例13: TestCopyTo

 public virtual void  TestCopyTo()
 {
     Token t = new Token();
     Token copy = (Token) TestSimpleAttributeImpls.AssertCopyIsEqual(t);
     Assert.AreEqual("", t.Term);
     Assert.AreEqual("", copy.Term);
     
     t = new Token(0, 5);
     char[] content = "hello".ToCharArray();
     t.SetTermBuffer(content, 0, 5);
     char[] buf = t.TermBuffer();
     copy = (Token) TestSimpleAttributeImpls.AssertCopyIsEqual(t);
     Assert.AreEqual(t.Term, copy.Term);
     Assert.AreNotSame(buf, copy.TermBuffer());
     
     Payload pl = new Payload(new byte[]{1, 2, 3, 4});
     t.Payload = pl;
     copy = (Token) TestSimpleAttributeImpls.AssertCopyIsEqual(t);
     Assert.AreEqual(pl, copy.Payload);
     Assert.AreNotSame(pl, copy.Payload);
 }
开发者ID:Nangal,项目名称:lucene.net,代码行数:21,代码来源:TestToken.cs

示例14: TestMixedStringArray

 public virtual void  TestMixedStringArray()
 {
     Token t = new Token("hello", 0, 5);
     Assert.AreEqual(t.TermLength(), 5);
     Assert.AreEqual(t.Term, "hello");
     t.SetTermBuffer("hello2");
     Assert.AreEqual(t.TermLength(), 6);
     Assert.AreEqual(t.Term, "hello2");
     t.SetTermBuffer("hello3".ToCharArray(), 0, 6);
     Assert.AreEqual(t.Term, "hello3");
     
     char[] buffer = t.TermBuffer();
     buffer[1] = 'o';
     Assert.AreEqual(t.Term, "hollo3");
 }
开发者ID:Nangal,项目名称:lucene.net,代码行数:15,代码来源:TestToken.cs

示例15: SetCurrentToken

 private void SetCurrentToken(Token token)
 {
     if (token == null) return;
     ClearAttributes();
     _termAtt.SetTermBuffer(token.TermBuffer(), 0, token.TermLength());
     _posIncrAtt.SetPositionIncrement(token.GetPositionIncrement());
     _flagsAtt.SetFlags(token.GetFlags());
     _offsetAtt.SetOffset(token.StartOffset(), token.EndOffset());
     _typeAtt.SetType(token.Type());
     _payloadAtt.SetPayload(token.GetPayload());
 }
开发者ID:kstenson,项目名称:NHibernate.Search,代码行数:11,代码来源:PrefixAwareTokenStream.cs


注:本文中的Lucene.Net.Analysis.Token.TermBuffer方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。