本文整理汇总了C#中Tokenizer.Consume方法的典型用法代码示例。如果您正苦于以下问题：C# Tokenizer.Consume方法的具体用法？C# Tokenizer.Consume怎么用？C# Tokenizer.Consume使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类Tokenizer的用法示例。
在下文中一共展示了Tokenizer.Consume方法的8个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的C#代码示例。
示例1: IsMatchImpl
protected override Token IsMatchImpl( Tokenizer tokenizer )
{
var str = new StringBuilder();
if( tokenizer.Current == StringDelim )
{
tokenizer.Consume();
while( !tokenizer.EndOfStream() && tokenizer.Current != StringDelim )
{
str.Append( tokenizer.Current );
tokenizer.Consume();
}
if( tokenizer.Current == StringDelim )
{
tokenizer.Consume();
}
}
if( str.Length > 0 )
{
return new Token( TokenType.QuotedString, str.ToString() );
}
return null;
}示例2: IsMatchImpl
protected override Token IsMatchImpl( Tokenizer tokenizer )
{
foreach( var character in Match )
{
if( tokenizer.Current == character.ToString( CultureInfo.InvariantCulture ) )
{
tokenizer.Consume();
}
else
{
return null;
}
}
bool found;
if( !AllowAsSubString )
{
var next = tokenizer.Current;
found = string.IsNullOrWhiteSpace( next ) || SpecialCharacters.Any( character => character.Match == next );
}
else
{
found = true;
}
if( found )
{
return new Token( myTokenType, Match );
}
return null;
}示例3: GetIntegers
private String GetIntegers( Tokenizer tokenizer )
{
string num = null;
while( tokenizer.Current != null && myRegex.IsMatch( tokenizer.Current ) )
{
num += tokenizer.Current;
tokenizer.Consume();
}
if( num != null )
{
return num;
}
return null;
}示例4: IsMatchImpl
protected override Token IsMatchImpl( Tokenizer tokenizer )
{
var str = new StringBuilder();
while( !tokenizer.EndOfStream() && String.IsNullOrWhiteSpace( tokenizer.Current ) )
{
str.Append( tokenizer.Current );
tokenizer.Consume();
}
if( str.Length > 0 )
{
return new Token( TokenType.WhiteSpace, str.ToString() );
}
return null;
}示例5: IsMatchImpl
protected override Token IsMatchImpl( Tokenizer tokenizer )
{
var str = new StringBuilder();
while( !tokenizer.EndOfStream() && ( tokenizer.Current == "\r" || tokenizer.Current == "\n" ) )
{
str.Append( tokenizer.Current );
tokenizer.Consume();
}
if( str.ToString() == Environment.NewLine )
{
return new Token( TokenType.NewLine );
}
return null;
}示例6: IsMatchImpl
protected override Token IsMatchImpl(Tokenizer tokenizer)
{
bool foundWhiteSpace = false;
while (!tokenizer.End() && String.IsNullOrWhiteSpace(tokenizer.Current))
{
foundWhiteSpace = true;
tokenizer.Consume();
}
if (foundWhiteSpace)
{
return new Token(TokenType.WhiteSpace);
}
return null;
}示例7: IsMatchImpl
protected override Token IsMatchImpl( Tokenizer tokenizer )
{
String current = null;
while( !tokenizer.EndOfStream() && !String.IsNullOrWhiteSpace( tokenizer.Current ) && mySpecialCharacters.All( m => m.Match != tokenizer.Current ) )
{
current += tokenizer.Current;
tokenizer.Consume();
}
if( current == null )
{
return null;
}
// can't start a word with a special character
if( mySpecialCharacters.Any( c => current.StartsWith( c.Match ) ) )
{
throw new Exception( String.Format( "Cannot start a word with a special character {0}", current ) );
}
return new Token( TokenType.Word, current );
}示例8: IsMatchImpl
protected override Token IsMatchImpl( Tokenizer tokenizer )
{
var leftOperand = GetIntegers( tokenizer );
if( leftOperand != null )
{
if( tokenizer.Current == "." )
{
tokenizer.Consume();
var rightOperand = GetIntegers( tokenizer );
// found a float
if( rightOperand != null )
{
return new Token( TokenType.Number, leftOperand + "." + rightOperand );
}
}
return new Token( TokenType.Number, leftOperand );
}
return null;
}