C# TokenStream.Reset方法代码示例

        public static void AssertTokenStreamContents(TokenStream ts, String[] output, Int32[] startOffsets, Int32[] endOffsets, String[] types, Int32[] posIncrements, Int32? finalOffset) {
            Assert.IsNotNull(output);
            var checkClearAtt = (CheckClearAttributesAttribute) ts.AddAttribute(typeof (CheckClearAttributesAttribute));

            Assert.IsTrue(ts.HasAttribute(typeof (TermAttribute)), "has no TermAttribute");
            var termAtt = (TermAttribute) ts.GetAttribute(typeof (TermAttribute));

            OffsetAttribute offsetAtt = null;
            if (startOffsets != null || endOffsets != null || finalOffset != null) {
                Assert.IsTrue(ts.HasAttribute(typeof (OffsetAttribute)), "has no OffsetAttribute");
                offsetAtt = (OffsetAttribute) ts.GetAttribute(typeof (OffsetAttribute));
            }

            TypeAttribute typeAtt = null;
            if (types != null) {
                Assert.IsTrue(ts.HasAttribute(typeof (TypeAttribute)), "has no TypeAttribute");
                typeAtt = (TypeAttribute) ts.GetAttribute(typeof (TypeAttribute));
            }

            PositionIncrementAttribute posIncrAtt = null;
            if (posIncrements != null) {
                Assert.IsTrue(ts.HasAttribute(typeof (PositionIncrementAttribute)), "has no PositionIncrementAttribute");
                posIncrAtt = (PositionIncrementAttribute) ts.GetAttribute(typeof (PositionIncrementAttribute));
            }

            ts.Reset();
            for (Int32 i = 0; i < output.Length; i++) {
                // extra safety to enforce, that the state is not preserved and also assign bogus values
                ts.ClearAttributes();
                termAtt.SetTermBuffer("bogusTerm");
                if (offsetAtt != null) offsetAtt.SetOffset(14584724, 24683243);
                if (typeAtt != null) typeAtt.SetType("bogusType");
                if (posIncrAtt != null) posIncrAtt.SetPositionIncrement(45987657);

                checkClearAtt.GetAndResetClearCalled(); // reset it, because we called clearAttribute() before
                Assert.IsTrue(ts.IncrementToken(), "token " + i + " does not exist");
                Assert.IsTrue(checkClearAtt.GetAndResetClearCalled(), "clearAttributes() was not called correctly in TokenStream chain");

                Assert.AreEqual(output[i], termAtt.Term(), "term " + i);
                if (startOffsets != null)
                    Assert.AreEqual(startOffsets[i], offsetAtt.StartOffset(), "startOffset " + i);
                if (endOffsets != null)
                    Assert.AreEqual(endOffsets[i], offsetAtt.EndOffset(), "endOffset " + i);
                if (types != null)
                    Assert.AreEqual(types[i], typeAtt.Type(), "type " + i);
                if (posIncrements != null)
                    Assert.AreEqual(posIncrements[i], posIncrAtt.GetPositionIncrement(), "posIncrement " + i);
            }
            Assert.IsFalse(ts.IncrementToken(), "end of stream");
            ts.End();
            if (finalOffset.HasValue)
                Assert.AreEqual(finalOffset.Value, offsetAtt.EndOffset(), "finalOffset ");
            ts.Close();
        }

 internal virtual void VerifyPayload(TokenStream ts)
 {
     IPayloadAttribute payloadAtt = ts.GetAttribute<IPayloadAttribute>();
     ts.Reset();
     for (sbyte b = 1; ; b++)
     {
         bool hasNext = ts.IncrementToken();
         if (!hasNext)
         {
             break;
         }
         // System.out.println("id="+System.identityHashCode(nextToken) + " " + t);
         // System.out.println("payload=" + (int)nextToken.getPayload().toByteArray()[0]);
         assertEquals(b, payloadAtt.Payload.Bytes[0]);
     }
 }

 protected void Create(string text)
 {
     _tokenStream = new TokenStream();
     _tokenStream.Reset(text);
 }

        // we only check a few core attributes here.
        // TODO: test other things
        public virtual void assertEquals(string s, TokenStream left, TokenStream right)
        {
            left.Reset();
            right.Reset();
            ICharTermAttribute leftTerm = left.AddAttribute<ICharTermAttribute>();
            ICharTermAttribute rightTerm = right.AddAttribute<ICharTermAttribute>();
            IOffsetAttribute leftOffset = left.AddAttribute<IOffsetAttribute>();
            IOffsetAttribute rightOffset = right.AddAttribute<IOffsetAttribute>();
            IPositionIncrementAttribute leftPos = left.AddAttribute<IPositionIncrementAttribute>();
            IPositionIncrementAttribute rightPos = right.AddAttribute<IPositionIncrementAttribute>();

            while (left.IncrementToken())
            {
                assertTrue("wrong number of tokens for input: " + s, right.IncrementToken());
                assertEquals("wrong term text for input: " + s, leftTerm.ToString(), rightTerm.ToString());
                assertEquals("wrong position for input: " + s, leftPos.PositionIncrement, rightPos.PositionIncrement);
                assertEquals("wrong start offset for input: " + s, leftOffset.StartOffset(), rightOffset.StartOffset());
                assertEquals("wrong end offset for input: " + s, leftOffset.EndOffset(), rightOffset.EndOffset());
            };
            assertFalse("wrong number of tokens for input: " + s, right.IncrementToken());
            left.End();
            right.End();
            assertEquals("wrong final offset for input: " + s, leftOffset.EndOffset(), rightOffset.EndOffset());
            left.Dispose();
            right.Dispose();
        }

        /// <summary>
        /// TODO: rewrite tests not to use string comparison.
        /// </summary>
        private static string tsToString(TokenStream @in)
        {
            StringBuilder @out = new StringBuilder();
            ICharTermAttribute termAtt = @in.AddAttribute<ICharTermAttribute>();
            // extra safety to enforce, that the state is not preserved and also
            // assign bogus values
            @in.ClearAttributes();
            termAtt.SetEmpty().Append("bogusTerm");
            @in.Reset();
            while (@in.IncrementToken())
            {
                if (@out.Length > 0)
                {
                    @out.Append(' ');
                }
                @out.Append(termAtt.ToString());
                @in.ClearAttributes();
                termAtt.SetEmpty().Append("bogusTerm");
            }

            @in.Dispose();
            return @out.ToString();
        }