当前位置: 首页>>代码示例>>C#>>正文

C# Tokenizer.GetEnumerator方法代码示例

本文整理汇总了C#中Tokenizer.GetEnumerator方法的典型用法代码示例。如果您正苦于以下问题：C# Tokenizer.GetEnumerator方法的具体用法？C# Tokenizer.GetEnumerator怎么用？C# Tokenizer.GetEnumerator使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类Tokenizer的用法示例。


在下文中一共展示了Tokenizer.GetEnumerator方法的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的C#代码示例。

示例1: AnTempalteWithOutAnySubsitutionsShouldBeReturnedAsSuch

 public void AnTempalteWithOutAnySubsitutionsShouldBeReturnedAsSuch()
 {
     const string INPUT = "abcdefgh";
     var tokenizer = new Tokenizer(INPUT, '\\', new[] {"."}, null);
     IEnumerator<Token> enumerator = tokenizer.GetEnumerator();
     Assert.IsTrue(enumerator.MoveNext());
     Assert.That(INPUT, Is.EqualTo(enumerator.Current.Contents));
     Assert.IsFalse(enumerator.MoveNext());
 }
开发者ID:rslijp，项目名称:sharptiles，代码行数:9，代码来源:TokenizerTest.cs

示例2: DoubleWhiteSpaceSurroundedSeperatorsShouldBeFoundWhenSurroundedByWhiteSpacesAtEndOfInput

 public void DoubleWhiteSpaceSurroundedSeperatorsShouldBeFoundWhenSurroundedByWhiteSpacesAtEndOfInput()
 {
     const string PART1 = "abc";
     const string SEP = ",";
     const string INPUT = PART1 + " " + SEP + " " + SEP;
     var tokenizer = new Tokenizer(INPUT, true, '\\', new[] {"."}, null, new[] {","});
     IEnumerator<Token> enumerator = tokenizer.GetEnumerator();
     Assert.IsTrue(enumerator.MoveNext());
     Assert.That(enumerator.Current.Type, Is.EqualTo(TokenType.Regular));
     Assert.That(enumerator.Current.Contents, Is.EqualTo(PART1));
     Assert.IsTrue(enumerator.MoveNext());
     Assert.That(enumerator.Current.Type, Is.EqualTo(TokenType.Seperator));
     Assert.That(enumerator.Current.Contents, Is.EqualTo(SEP));
     Assert.IsTrue(enumerator.MoveNext());
     Assert.That(enumerator.Current.Type, Is.EqualTo(TokenType.Seperator));
     Assert.That(enumerator.Current.Contents, Is.EqualTo(SEP));
     Assert.IsFalse(enumerator.MoveNext());
 }
开发者ID:rslijp，项目名称:sharptiles，代码行数:18，代码来源:TokenizerTest.cs

示例3: TokenizerShouldIgnoreEscapedLiteralInEmbeddedInLiterals

 public void TokenizerShouldIgnoreEscapedLiteralInEmbeddedInLiterals()
 {
     const string INPUT = "abc'\\''defgh";
     var tokenizer = new Tokenizer(INPUT, '\\', new[] {"."}, new[] {'\''});
     IEnumerator<Token> enumerator = tokenizer.GetEnumerator();
     Assert.IsTrue(enumerator.MoveNext());
     Assert.That(enumerator.Current.Contents, Is.EqualTo("abc'''defgh"));
     Assert.IsFalse(enumerator.MoveNext());
 }
开发者ID:rslijp，项目名称:sharptiles，代码行数:9，代码来源:TokenizerTest.cs

示例4: WhiteSpaceSurroundedSeperatorsShouldNotBeReturnedIfNotAsked

 public void WhiteSpaceSurroundedSeperatorsShouldNotBeReturnedIfNotAsked()
 {
     const string PART1 = "xs";
     const string PART2 = "ec";
     const string SEP = ",";
     const string INPUT = PART1 + SEP + " " + SEP + " " + SEP + " " + PART2;
     var tokenizer = new Tokenizer(INPUT, false, '\\', new[] {"."}, null, new[] {","});
     IEnumerator<Token> enumerator = tokenizer.GetEnumerator();
     Assert.IsTrue(enumerator.MoveNext());
     Assert.That(enumerator.Current.Type, Is.EqualTo(TokenType.Regular));
     Assert.That(enumerator.Current.Contents, Is.EqualTo(PART1 + SEP));
     Assert.IsTrue(enumerator.MoveNext());
     Assert.That(enumerator.Current.Type, Is.EqualTo(TokenType.Regular));
     Assert.That(enumerator.Current.Contents, Is.EqualTo(PART2));
     Assert.IsFalse(enumerator.MoveNext());
 }
开发者ID:rslijp，项目名称:sharptiles，代码行数:16，代码来源:TokenizerTest.cs

示例5: WhiteSpaceSurroundedSeperatorsOnly

 public void WhiteSpaceSurroundedSeperatorsOnly()
 {
     const string INPUT = ",";
     var tokenizer = new Tokenizer(INPUT, true, '\\', new[] {"."}, null, new[] {","});
     IEnumerator<Token> enumerator = tokenizer.GetEnumerator();
     Assert.IsTrue(enumerator.MoveNext());
     Assert.That(enumerator.Current.Contents, Is.EqualTo(INPUT));
     Assert.That(enumerator.Current.Type, Is.EqualTo(TokenType.Seperator));
     Assert.IsFalse(enumerator.MoveNext());
 }
开发者ID:rslijp，项目名称:sharptiles，代码行数:10，代码来源:TokenizerTest.cs

示例6: WhenAskedTokenizerShouldReturnSeperator

        public void WhenAskedTokenizerShouldReturnSeperator()
        {
            const string PART1 = "abc";
            const string PART2 = "defg";
            const string PART3 = "hijklmnop";
            const string INPUT = PART1 + "\\." + PART2 + "." + PART3 + ".\\.";

            var tokenizer = new Tokenizer(INPUT, true, '\\', new[] {"."}, null);
            IEnumerator<Token> enumerator = tokenizer.GetEnumerator();
            Assert.IsTrue(enumerator.MoveNext());
            Assert.That(enumerator.Current.Contents, Is.EqualTo(PART1 + "." + PART2));
            Assert.IsTrue(enumerator.MoveNext());
            Assert.That(enumerator.Current.Contents, Is.EqualTo("."));
            Assert.IsTrue(enumerator.MoveNext());
            Assert.That(enumerator.Current.Contents, Is.EqualTo(PART3));
            Assert.IsTrue(enumerator.MoveNext());
            Assert.That(enumerator.Current.Contents, Is.EqualTo("."));
            Assert.IsTrue(enumerator.MoveNext());
            Assert.That(enumerator.Current.Contents, Is.EqualTo("."));
            Assert.IsFalse(enumerator.MoveNext());
            Assert.That(enumerator.Current, Is.Null);
        }
开发者ID:rslijp，项目名称:sharptiles，代码行数:22，代码来源:TokenizerTest.cs

示例7: TokenizerShouldReturnsLiteralsWhenAskedAtStartAndEnd

        public void TokenizerShouldReturnsLiteralsWhenAskedAtStartAndEnd()
        {
            const string PART2 = "defg";
            const string PART3 = "hijklmnop";
            const string PART4 = "qrst";
            const string INPUT = "'" + PART2 + "'" + PART3 + "'" + PART4 + "'";

            var tokenizer = new Tokenizer(INPUT, true, true, '\\', new[] {"."}, new[] {'\''}, null);
            IEnumerator<Token> enumerator = tokenizer.GetEnumerator();
            Assert.IsTrue(enumerator.MoveNext());
            Assert.That(enumerator.Current.Contents, Is.EqualTo(PART2));
            Assert.That(enumerator.Current.Type, Is.EqualTo(TokenType.Literal));
            Assert.IsTrue(enumerator.MoveNext());
            Assert.That(enumerator.Current.Contents, Is.EqualTo(PART3));
            Assert.That(enumerator.Current.Type, Is.EqualTo(TokenType.Regular));
            Assert.IsTrue(enumerator.MoveNext());
            Assert.That(enumerator.Current.Contents, Is.EqualTo(PART4));
            Assert.That(enumerator.Current.Type, Is.EqualTo(TokenType.Literal));
            Assert.IsFalse(enumerator.MoveNext());
            Assert.That(enumerator.Current, Is.Null);
        }
开发者ID:rslijp，项目名称:sharptiles，代码行数:21，代码来源:TokenizerTest.cs

示例8: TokenizerShouldNotIgnoreOtherTokensBesidesTheEmbeddedOnInLiterals

 public void TokenizerShouldNotIgnoreOtherTokensBesidesTheEmbeddedOnInLiterals()
 {
     const string PART1 = "abc'.'defgh";
     const string PART2 = "wxz";
     const string INPUT = PART1 + "." + PART2;
     var tokenizer = new Tokenizer(INPUT, '\\', new[] {"."}, new[] {'\''});
     IEnumerator<Token> enumerator = tokenizer.GetEnumerator();
     Assert.IsTrue(enumerator.MoveNext());
     Assert.That(enumerator.Current.Contents, Is.EqualTo(PART1));
     Assert.IsTrue(enumerator.MoveNext());
     Assert.That(enumerator.Current.Contents, Is.EqualTo(PART2));
     Assert.IsFalse(enumerator.MoveNext());
 }
开发者ID:rslijp，项目名称:sharptiles，代码行数:13，代码来源:TokenizerTest.cs

示例9: SeperatorAndRegularTokensShouldBeDistinctable

        public void SeperatorAndRegularTokensShouldBeDistinctable()
        {
            const string PART1 = "abc";
            const string PART2 = "defg";
            const string PART3 = "hijklmnop";
            const string INPUT = PART1 + "\\." + PART2 + "." + PART3 + ".\\.";

            var tokenizer = new Tokenizer(INPUT, true, '\\', new[] {"."}, null);
            IEnumerator<Token> enumerator = tokenizer.GetEnumerator();
            Assert.IsTrue(enumerator.MoveNext());
            Assert.That(enumerator.Current.Type, Is.EqualTo(TokenType.Regular));
            Assert.IsTrue(enumerator.MoveNext());
            Assert.That(enumerator.Current.Type, Is.EqualTo(TokenType.Seperator));
            Assert.IsTrue(enumerator.MoveNext());
            Assert.That(enumerator.Current.Type, Is.EqualTo(TokenType.Regular));
            Assert.IsTrue(enumerator.MoveNext());
            Assert.That(enumerator.Current.Type, Is.EqualTo(TokenType.Seperator));
            Assert.IsTrue(enumerator.MoveNext());
            Assert.That(enumerator.Current.Type, Is.EqualTo(TokenType.Regular));
            Assert.IsFalse(enumerator.MoveNext());
            Assert.That(enumerator.Current, Is.Null);
        }
开发者ID:rslijp，项目名称:sharptiles，代码行数:22，代码来源:TokenizerTest.cs

示例10: ResetShouldRestartEnumerator

 public void ResetShouldRestartEnumerator()
 {
     const string INPUT = "abcdefgh";
     var tokenizer = new Tokenizer(INPUT, '\\', new[] {"."}, null);
     IEnumerator<Token> enumerator = tokenizer.GetEnumerator();
     Assert.IsTrue(enumerator.MoveNext());
     Assert.IsFalse(enumerator.MoveNext());
     enumerator.Reset();
     Assert.IsTrue(enumerator.MoveNext());
     Assert.IsFalse(enumerator.MoveNext());
 }
开发者ID:rslijp，项目名称:sharptiles，代码行数:11，代码来源:TokenizerTest.cs

示例11: MissingEscapedCharactersShouldThrowAnException

        public void MissingEscapedCharactersShouldThrowAnException()
        {
            const string INPUT = "abc\\";

            var tokenizer = new Tokenizer(INPUT, '\\', new[] {"."}, null);
            IEnumerator<Token> enumerator = tokenizer.GetEnumerator();
            try
            {
                enumerator.MoveNext();
            }
            catch (TokenException Te)
            {
                Assert.That(Te.MessageWithOutContext, Is.EqualTo(TokenException.MoreCharactersExpectedAt(4).Message));
            }
        }
开发者ID:rslijp，项目名称:sharptiles，代码行数:15，代码来源:TokenizerTest.cs

示例12: IllegalEscapedCharactersShouldThrowAnException

        public void IllegalEscapedCharactersShouldThrowAnException()
        {
            const string INPUT = "abc\\a";

            var tokenizer = new Tokenizer(INPUT, '\\', new[] {"."}, null);
            IEnumerator<Token> enumerator = tokenizer.GetEnumerator();
            try
            {
                enumerator.MoveNext();
            }
            catch (TokenException Te)
            {
                Assert.That(Te.Message, Is.EqualTo(TokenException.IllegalEscapeCharacter('a').Message));
            }
        }
开发者ID:rslijp，项目名称:sharptiles，代码行数:15，代码来源:TokenizerTest.cs

示例13: EscapedEscapreCharactersShouldBeIgnored

        public void EscapedEscapreCharactersShouldBeIgnored()
        {
            const string PART1 = "abc";
            const string PART2 = "defg";
            const string INPUT = PART1 + "\\\\" + PART2;

            var tokenizer = new Tokenizer(INPUT, '\\', new[] {"."}, null);
            IEnumerator<Token> enumerator = tokenizer.GetEnumerator();
            Assert.IsTrue(enumerator.MoveNext());
            Assert.That(enumerator.Current.Contents, Is.EqualTo(PART1 + "\\" + PART2));
            Assert.IsFalse(enumerator.MoveNext());
            Assert.That(enumerator.Current, Is.Null);
        }
开发者ID:rslijp，项目名称:sharptiles，代码行数:13，代码来源:TokenizerTest.cs

示例14: EmptyStringShouldNotCollapseTokenizer

 public void EmptyStringShouldNotCollapseTokenizer()
 {
     const string INPUT = "";
     var tokenizer = new Tokenizer(INPUT, '\\', new[] {"."}, null);
     IEnumerator<Token> enumerator = tokenizer.GetEnumerator();
     Assert.IsFalse(enumerator.MoveNext());
 }
开发者ID:rslijp，项目名称:sharptiles，代码行数:7，代码来源:TokenizerTest.cs

示例15: TokenizerShouldIgnoreNestedLiteral

 public void TokenizerShouldIgnoreNestedLiteral()
 {
     const string PART1 = "abc'.\"'defgh";
     const string PART2 = "wxz";
     const string INPUT = PART1 + "." + PART2;
     var tokenizer = new Tokenizer(INPUT, '\\', new[] {"."}, new[] {'\'', '"'});
     IEnumerator<Token> enumerator = tokenizer.GetEnumerator();
     Assert.IsTrue(enumerator.MoveNext());
     Assert.That(enumerator.Current.Contents, Is.EqualTo(PART1));
     Assert.IsTrue(enumerator.MoveNext());
     Assert.That(enumerator.Current.Contents, Is.EqualTo(PART2));
     Assert.IsFalse(enumerator.MoveNext());
 }
开发者ID:rslijp，项目名称:sharptiles，代码行数:13，代码来源:TokenizerTest.cs


注：本文中的Tokenizer.GetEnumerator方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台，相关代码片段筛选自各路编程大神贡献的开源项目，源码版权归原作者所有，传播和使用请参考对应项目的License；未经允许，请勿转载。