本文整理汇总了C#中Tokenizer.incrementToken方法的典型用法代码示例。如果您正苦于以下问题：C# Tokenizer.incrementToken方法的具体用法？C# Tokenizer.incrementToken怎么用？C# Tokenizer.incrementToken使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类Tokenizer的用法示例。
在下文中一共展示了Tokenizer.incrementToken方法的4个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的C#代码示例。
示例1: testRandom
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testRandom() throws Exception
public virtual void testRandom()
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final int alphabetSize = org.apache.lucene.util.TestUtil.nextInt(random(), 2, 7);
int alphabetSize = TestUtil.Next(random(), 2, 7);
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final int docLen = atLeast(3000);
int docLen = atLeast(3000);
//final int docLen = 50;
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final String document = getRandomString('a', alphabetSize, docLen);
string document = getRandomString('a', alphabetSize, docLen);
if (VERBOSE)
{
Console.WriteLine("TEST: doc=" + document);
}
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final int numSyn = atLeast(5);
int numSyn = atLeast(5);
//final int numSyn = 2;
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final java.util.Map<String,OneSyn> synMap = new java.util.HashMap<>();
IDictionary<string, OneSyn> synMap = new Dictionary<string, OneSyn>();
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final java.util.List<OneSyn> syns = new java.util.ArrayList<>();
IList<OneSyn> syns = new List<OneSyn>();
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final boolean dedup = random().nextBoolean();
bool dedup = random().nextBoolean();
if (VERBOSE)
{
Console.WriteLine(" dedup=" + dedup);
}
b = new SynonymMap.Builder(dedup);
for (int synIDX = 0;synIDX < numSyn;synIDX++)
{
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final String synIn = getRandomString('a', alphabetSize, org.apache.lucene.util.TestUtil.nextInt(random(), 1, 5)).trim();
string synIn = getRandomString('a', alphabetSize, TestUtil.Next(random(), 1, 5)).Trim();
OneSyn s = synMap[synIn];
if (s == null)
{
s = new OneSyn();
[email protected] = synIn;
syns.Add(s);
[email protected] = new List<>();
synMap[synIn] = s;
s.keepOrig = random().nextBoolean();
}
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final String synOut = getRandomString('0', 10, org.apache.lucene.util.TestUtil.nextInt(random(), 1, 5)).trim();
string synOut = getRandomString('0', 10, TestUtil.Next(random(), 1, 5)).Trim();
[email protected](synOut);
add(synIn, synOut, s.keepOrig);
if (VERBOSE)
{
Console.WriteLine(" syns[" + synIDX + "] = " + [email protected] + " -> " + [email protected] + " keepOrig=" + s.keepOrig);
}
}
tokensIn = new MockTokenizer(new StringReader("a"), MockTokenizer.WHITESPACE, true);
tokensIn.reset();
assertTrue(tokensIn.incrementToken());
assertFalse(tokensIn.incrementToken());
tokensIn.end();
tokensIn.close();
tokensOut = new SynonymFilter(tokensIn, b.build(), true);
termAtt = tokensOut.addAttribute(typeof(CharTermAttribute));
posIncrAtt = tokensOut.addAttribute(typeof(PositionIncrementAttribute));
posLenAtt = tokensOut.addAttribute(typeof(PositionLengthAttribute));
offsetAtt = tokensOut.addAttribute(typeof(OffsetAttribute));
if (dedup)
{
pruneDups(syns);
}
//JAVA TO C# CONVERTER WARNING: The original Java variable was marked 'final':
//ORIGINAL LINE: final String expected = slowSynMatcher(document, syns, 5);
string expected = slowSynMatcher(document, syns, 5);
if (VERBOSE)
{
Console.WriteLine("TEST: expected=" + expected);
}
verify(document, expected);
}示例2: testBasic2
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testBasic2() throws Exception
public virtual void testBasic2()
{
b = new SynonymMap.Builder(true);
const bool keepOrig = false;
add("aaa", "aaaa1 aaaa2 aaaa3", keepOrig);
add("bbb", "bbbb1 bbbb2", keepOrig);
tokensIn = new MockTokenizer(new StringReader("a"), MockTokenizer.WHITESPACE, true);
tokensIn.reset();
assertTrue(tokensIn.incrementToken());
assertFalse(tokensIn.incrementToken());
tokensIn.end();
tokensIn.close();
tokensOut = new SynonymFilter(tokensIn, b.build(), true);
termAtt = tokensOut.addAttribute(typeof(CharTermAttribute));
posIncrAtt = tokensOut.addAttribute(typeof(PositionIncrementAttribute));
posLenAtt = tokensOut.addAttribute(typeof(PositionLengthAttribute));
offsetAtt = tokensOut.addAttribute(typeof(OffsetAttribute));
if (keepOrig)
{
verify("xyzzy bbb pot of gold", "xyzzy bbb/bbbb1 pot/bbbb2 of gold");
verify("xyzzy aaa pot of gold", "xyzzy aaa/aaaa1 pot/aaaa2 of/aaaa3 gold");
}
else
{
verify("xyzzy bbb pot of gold", "xyzzy bbbb1 pot/bbbb2 of gold");
verify("xyzzy aaa pot of gold", "xyzzy aaaa1 pot/aaaa2 of/aaaa3 gold");
}
}示例3: testOutputHangsOffEnd
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testOutputHangsOffEnd() throws Exception
public virtual void testOutputHangsOffEnd()
{
b = new SynonymMap.Builder(true);
const bool keepOrig = false;
// b hangs off the end (no input token under it):
add("a", "a b", keepOrig);
tokensIn = new MockTokenizer(new StringReader("a"), MockTokenizer.WHITESPACE, true);
tokensIn.reset();
assertTrue(tokensIn.incrementToken());
assertFalse(tokensIn.incrementToken());
tokensIn.end();
tokensIn.close();
tokensOut = new SynonymFilter(tokensIn, b.build(), true);
termAtt = tokensOut.addAttribute(typeof(CharTermAttribute));
posIncrAtt = tokensOut.addAttribute(typeof(PositionIncrementAttribute));
offsetAtt = tokensOut.addAttribute(typeof(OffsetAttribute));
posLenAtt = tokensOut.addAttribute(typeof(PositionLengthAttribute));
// Make sure endOffset inherits from previous input token:
verify("a", "a b:1");
}示例4: testBasic
//JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
//ORIGINAL LINE: public void testBasic() throws Exception
public virtual void testBasic()
{
b = new SynonymMap.Builder(true);
add("a", "foo", true);
add("a b", "bar fee", true);
add("b c", "dog collar", true);
add("c d", "dog harness holder extras", true);
add("m c e", "dog barks loudly", false);
add("i j k", "feep", true);
add("e f", "foo bar", false);
add("e f", "baz bee", false);
add("z", "boo", false);
add("y", "bee", true);
tokensIn = new MockTokenizer(new StringReader("a"), MockTokenizer.WHITESPACE, true);
tokensIn.reset();
assertTrue(tokensIn.incrementToken());
assertFalse(tokensIn.incrementToken());
tokensIn.end();
tokensIn.close();
tokensOut = new SynonymFilter(tokensIn, b.build(), true);
termAtt = tokensOut.addAttribute(typeof(CharTermAttribute));
posIncrAtt = tokensOut.addAttribute(typeof(PositionIncrementAttribute));
posLenAtt = tokensOut.addAttribute(typeof(PositionLengthAttribute));
offsetAtt = tokensOut.addAttribute(typeof(OffsetAttribute));
verify("a b c", "a/bar b/fee c");
// syn output extends beyond input tokens
verify("x a b c d", "x a/bar b/fee c/dog d/harness holder extras");
verify("a b a", "a/bar b/fee a/foo");
// outputs that add to one another:
verify("c d c d", "c/dog d/harness c/holder/dog d/extras/harness holder extras");
// two outputs for same input
verify("e f", "foo/baz bar/bee");
// verify multi-word / single-output offsets:
verify("g i j k g", "g i/feep:7_3 j k g");
// mixed keepOrig true/false:
verify("a m c e x", "a/foo dog barks loudly x");
verify("c d m c e x", "c/dog d/harness holder/dog extras/barks loudly x");
assertTrue(tokensOut.CaptureCount > 0);
// no captureStates when no syns matched
verify("p q r s t", "p q r s t");
assertEquals(0, tokensOut.CaptureCount);
// no captureStates when only single-input syns, w/ no
// lookahead needed, matched
verify("p q z y t", "p q boo y/bee t");
assertEquals(0, tokensOut.CaptureCount);
}