本文整理汇总了C#中Lucene.Net.Analysis.Token类的典型用法代码示例。如果您正苦于以下问题：C# Token类的具体用法？C# Token怎么用？C# Token使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
Token类属于Lucene.Net.Analysis命名空间，在下文中一共展示了Token类的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的C#代码示例。
示例1: TestCtor
public virtual void TestCtor()
{
Token t = new Token();
char[] content = "hello".ToCharArray();
t.SetTermBuffer(content, 0, content.Length);
char[] buf = t.TermBuffer();
Assert.AreNotEqual(t.TermBuffer(), content);
Assert.AreEqual("hello", t.Term);
Assert.AreEqual("word", t.Type);
Assert.AreEqual(0, t.Flags);
t = new Token(6, 22);
t.SetTermBuffer(content, 0, content.Length);
Assert.AreEqual("hello", t.Term);
Assert.AreEqual("(hello,6,22)", t.ToString());
Assert.AreEqual("word", t.Type);
Assert.AreEqual(0, t.Flags);
t = new Token(6, 22, 7);
t.SetTermBuffer(content, 0, content.Length);
Assert.AreEqual("hello", t.Term);
Assert.AreEqual("(hello,6,22)", t.ToString());
Assert.AreEqual(7, t.Flags);
t = new Token(6, 22, "junk");
t.SetTermBuffer(content, 0, content.Length);
Assert.AreEqual("hello", t.Term);
Assert.AreEqual("(hello,6,22,type=junk)", t.ToString());
Assert.AreEqual(0, t.Flags);
}示例2: GetTokenPositioner
public override TokenPositioner GetTokenPositioner(Token token)
{
return
token.GetPositionIncrement() == 0
? TokenPositioner.NewRow
: TokenPositioner.NewColumn;
}开发者ID:kstenson，项目名称:NHibernate.Search，代码行数:7，代码来源:TwoDimensionalNonWeightedSynonymTokenSettingsCodec.cs
示例3: Next
public override Token Next(Token reusableToken)
{
System.Diagnostics.Debug.Assert(reusableToken != null);
Token nextToken = input.Next(reusableToken);
sink.Add(nextToken);
return nextToken;
}示例4: NextToken
public static Token NextToken(TokenStream input, Token reusableToken)
{
if (input == null)
return null;
if (!input.IncrementToken())
return null;
ITermAttribute termAtt = input.GetAttribute<ITermAttribute>();
IOffsetAttribute offsetAtt = input.GetAttribute<IOffsetAttribute>();
ITypeAttribute typeAtt = input.GetAttribute<ITypeAttribute>();
if (reusableToken == null)
{
reusableToken = new Token();
}
reusableToken.Clear();
if (termAtt != null)
reusableToken.SetTermBuffer(termAtt.TermBuffer(), 0, termAtt.TermLength());
if (offsetAtt != null)
{
reusableToken.StartOffset = offsetAtt.StartOffset;
reusableToken.EndOffset = offsetAtt.EndOffset;
}
if (typeAtt != null)
reusableToken.Type = typeAtt.Type;
return reusableToken;
}示例5: Add
public override void Add(Token t)
{
if (t != null && t.Term().ToUpper().Equals("The".ToUpper()))
{
base.Add(t);
}
}示例6: Next
public override Token Next(/* in */ Token reusableToken)
{
System.Diagnostics.Debug.Assert(reusableToken != null);
Token nextToken = input.Next(reusableToken);
if (nextToken != null)
{
char[] buffer = nextToken.TermBuffer();
int length = nextToken.TermLength();
// If no characters actually require rewriting then we
// just return token as-is:
for (int i = 0; i < length; i++)
{
char c = buffer[i];
if (c >= '\u00c0' && c <= '\ufb06')
{
RemoveAccents(buffer, length);
nextToken.SetTermBuffer(output, 0, outputPos);
break;
}
}
return nextToken;
}
else
return null;
}示例7: GetWeight
/// <summary>
/// Returns a 32 bit float from the payload, or 1f it null.
/// </summary>
/// <param name="token"></param>
/// <returns></returns>
public override float GetWeight(Token token)
{
if (token.GetPayload() == null || token.GetPayload().GetData() == null)
return 1f;
return PayloadHelper.DecodeFloat(token.GetPayload().GetData());
}示例8: SetWeight
/// <summary>
/// Stores a 32 bit float in the payload, or set it to null if 1f;
/// </summary>
/// <param name="token"></param>
/// <param name="weight"></param>
public override void SetWeight(Token token, float weight)
{
token.SetPayload(
weight == 1f
? null
: new Payload(PayloadHelper.EncodeFloat(weight))
);
}示例9: CutLeterDigitFilter
public CutLeterDigitFilter(TokenStream input)
: base(input)
{
reusableToken = new Token();
termAtt = AddAttribute<ITermAttribute>();
offsetAtt = AddAttribute<IOffsetAttribute>();
typeAtt = AddAttribute<ITypeAttribute>();
}示例10: Next
public override Token Next()
{
Token t = this.input.Next();
if (t != null)
{
t = new Token(_replaceDiacritics(t.TermText()), t.StartOffset(), t.EndOffset()/*, "DiacriticFiltered"*/);
}
return t;
}示例11: TestToString
public virtual void TestToString()
{
char[] b = new char[]{'a', 'l', 'o', 'h', 'a'};
Token t = new Token("", 0, 5);
t.SetTermBuffer(b, 0, 5);
Assert.AreEqual("(aloha,0,5)", t.ToString());
t.SetTermText("hi there");
Assert.AreEqual("(hi there,0,5)", t.ToString());
}示例12: AddToken
void AddToken(Token oriToken, int termBufferOffset, int termBufferLength, byte type)
{
Token token = new Token(oriToken.TermBuffer(), termBufferOffset, termBufferLength,
oriToken.StartOffset + termBufferOffset, oriToken.StartOffset + termBufferOffset + termBufferLength);
if (type == (byte)UnicodeCategory.DecimalDigitNumber)
token.Type = Word.TYPE_DIGIT;
else
token.Type = Word.TYPE_LETTER;
tokenQueue.Enqueue(token);
}示例13: Next
public override Token Next(Token result)
{
result = input.Next(result);
if (result != null)
{
if (stemmer.Stem(result.TermBuffer(), 0, result.termLength))
result.SetTermBuffer(stemmer.GetResultBuffer(), 0, stemmer.GetResultLength());
return result;
}
else
return null;
}示例14: TestResize
public virtual void TestResize()
{
Token t = new Token();
char[] content = "hello".ToCharArray();
t.SetTermBuffer(content, 0, content.Length);
for (int i = 0; i < 2000; i++)
{
t.ResizeTermBuffer(i);
Assert.IsTrue(i <= t.TermBuffer().Length);
Assert.AreEqual("hello", t.Term);
}
}示例15: Next
public override Token Next(/* in */ Token reusableToken)
{
System.Diagnostics.Debug.Assert(reusableToken != null);
Token nextToken = input.Next(reusableToken);
if (nextToken == null)
return null;
if (stemmer.Stem(nextToken.TermBuffer(), 0, nextToken.TermLength()))
nextToken.SetTermBuffer(stemmer.GetResultBuffer(), 0, stemmer.GetResultLength());
return nextToken;
}