本文整理汇总了C#中Tokenizer.EndOfStream方法的典型用法代码示例。如果您正苦于以下问题：C# Tokenizer.EndOfStream方法的具体用法？C# Tokenizer.EndOfStream怎么用？C# Tokenizer.EndOfStream使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类Tokenizer的用法示例。
在下文中一共展示了Tokenizer.EndOfStream方法的5个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的C#代码示例。
示例1: IsMatchImpl
protected override Token IsMatchImpl( Tokenizer tokenizer )
{
var str = new StringBuilder();
if( tokenizer.Current == StringDelim )
{
tokenizer.Consume();
while( !tokenizer.EndOfStream() && tokenizer.Current != StringDelim )
{
str.Append( tokenizer.Current );
tokenizer.Consume();
}
if( tokenizer.Current == StringDelim )
{
tokenizer.Consume();
}
}
if( str.Length > 0 )
{
return new Token( TokenType.QuotedString, str.ToString() );
}
return null;
}示例2: IsMatchImpl
protected override Token IsMatchImpl( Tokenizer tokenizer )
{
var str = new StringBuilder();
while( !tokenizer.EndOfStream() && String.IsNullOrWhiteSpace( tokenizer.Current ) )
{
str.Append( tokenizer.Current );
tokenizer.Consume();
}
if( str.Length > 0 )
{
return new Token( TokenType.WhiteSpace, str.ToString() );
}
return null;
}示例3: IsMatchImpl
protected override Token IsMatchImpl( Tokenizer tokenizer )
{
var str = new StringBuilder();
while( !tokenizer.EndOfStream() && ( tokenizer.Current == "\r" || tokenizer.Current == "\n" ) )
{
str.Append( tokenizer.Current );
tokenizer.Consume();
}
if( str.ToString() == Environment.NewLine )
{
return new Token( TokenType.NewLine );
}
return null;
}示例4: IsMatch
public Token IsMatch( Tokenizer tokenizer )
{
if( tokenizer.EndOfStream() )
{
return new Token( TokenType.EndOfStream );
}
tokenizer.TakeSnapshot();
var match = IsMatchImpl( tokenizer );
if( match == null )
{
tokenizer.RollbackSnapshot();
}
else
{
tokenizer.CommitSnapshot();
}
return match;
}示例5: IsMatchImpl
protected override Token IsMatchImpl( Tokenizer tokenizer )
{
String current = null;
while( !tokenizer.EndOfStream() && !String.IsNullOrWhiteSpace( tokenizer.Current ) && mySpecialCharacters.All( m => m.Match != tokenizer.Current ) )
{
current += tokenizer.Current;
tokenizer.Consume();
}
if( current == null )
{
return null;
}
// can't start a word with a special character
if( mySpecialCharacters.Any( c => current.StartsWith( c.Match ) ) )
{
throw new Exception( String.Format( "Cannot start a word with a special character {0}", current ) );
}
return new Token( TokenType.Word, current );
}