C# ISourceStream.CreateToken方法代码示例

 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   string tokenText = string.Empty;
   while (true) {
     //Find next position
     var newPos = source.Text.IndexOfAny(_stopChars, source.PreviewPosition);
     if(newPos == -1) {
       if(IsSet(FreeTextOptions.AllowEof)) {
         source.PreviewPosition = source.Text.Length;
         return source.CreateToken(this.OutputTerminal);
       }  else
         return null;
     }
     if (newPos == source.PreviewPosition)   // DC
     {
         context.AddParserError("(DC) in TryMatch, newPos == source.PreviewPosition", new object[] {});
         break;                              // DC
     }
     tokenText += source.Text.Substring(source.PreviewPosition, newPos - source.PreviewPosition);
     source.PreviewPosition = newPos;
     //if it is escape, add escaped text and continue search
     if (CheckEscape(source, ref tokenText)) 
       continue;
     //check terminators
     if (CheckTerminators(source, ref tokenText))
       break; //from while (true)        
   }
   return source.CreateToken(this.OutputTerminal, tokenText);
 }

 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   bool isEscape = source.PreviewChar == EscapeChar && EscapeChar != NoEscape;
   if(isEscape) {
     //return a token containing only escaped char
     var value = source.NextPreviewChar.ToString(); 
     source.PreviewPosition += 2; 
     return source.CreateToken(this.OutputTerminal, value);  
   }
   var stopIndex = source.Text.IndexOfAny(_stopChars, source.Location.Position + 1);
   if (stopIndex == source.Location.Position) return null; 
   if (stopIndex < 0) stopIndex = source.Text.Length; 
   source.PreviewPosition = stopIndex;
   return source.CreateToken(this.OutputTerminal);
 }//method

        private Token MatchQuoted(ParsingContext context, ISourceStream source)
        {
            char quoteChar = source.PreviewChar;
            if ((quoteChar != '\'') && (quoteChar != '"'))
            {
                return null;
            }

            source.PreviewPosition++;
            while (!source.EOF())
            {
                if (source.PreviewChar == quoteChar)
                {
                    source.PreviewPosition++;
                    return source.CreateToken(this.OutputTerminal);
                }

                // Escaped?
                if (source.PreviewChar == '\\')
                {
                    // Consume next
                    ++source.PreviewPosition;
                }

                source.PreviewPosition++;
            }

            return context.CreateErrorToken("Unbalanced quoted string");
        }

 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   Match m = _expression.Match(source.Text, source.PreviewPosition);
   if (!m.Success || m.Index != source.PreviewPosition) 
     return null;
   source.PreviewPosition += m.Length;
   return source.CreateToken(this.OutputTerminal); 
 }

 private Token CreateIncompleteToken(ParsingContext context, ISourceStream source) {
   source.PreviewPosition = source.Text.Length;
   Token result = source.CreateToken(this.OutputTerminal);
   result.Flags |= TokenFlags.IsIncomplete;
   context.VsLineScanState.TerminalIndex = this.MultilineIndex;
   return result; 
 }

 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   char current = source.PreviewChar;
   if (!LineTerminators.Contains(current)) return null;
   //Treat \r\n as a single terminator
   bool doExtraShift = (current == '\r' && source.NextPreviewChar == '\n');
   source.PreviewPosition++; //main shift
   if (doExtraShift)
     source.PreviewPosition++;
   Token result = source.CreateToken(this.OutputTerminal);
   return result;
 }

 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   try {
     var textValue = ReadBody(context, source);
     if (textValue == null) return null; 
     var value = ConvertValue(context, textValue);
     return source.CreateToken(this.OutputTerminal, value);
   } catch(Exception ex) {
     //we throw exception in DsvLiteral when we cannot find a closing quote for quoted value
     return source.CreateErrorToken(ex.Message);
   }
 }//method

 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   if (!source.MatchSymbol(OpenTag, true)) return null;
   source.PreviewPosition += OpenTag.Length;
   var endPos = source.Text.IndexOf(CloseTag, source.PreviewPosition);
   string content; 
   if(endPos > 0) {
     content = source.Text.Substring(source.PreviewPosition, endPos - source.PreviewPosition); 
     source.PreviewPosition = endPos + CloseTag.Length;
   } else {
     content = source.Text.Substring(source.PreviewPosition, source.Text.Length - source.PreviewPosition); 
     source.PreviewPosition = source.Text.Length;
   }
   var token = source.CreateToken(this.OutputTerminal, content); 
   return token;      
 }

 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   bool isHeadingOrList = TermType == WikiTermType.Heading || TermType == WikiTermType.List;
   if(isHeadingOrList) {
       bool isAfterNewLine = (context.PreviousToken == null || context.PreviousToken.Terminal == Grammar.NewLine);
       if(!isAfterNewLine)  return null;
   }
   if(!source.MatchSymbol(OpenTag)) return null;
   source.PreviewPosition += OpenTag.Length;
   //For headings and lists require space after
   if(TermType == WikiTermType.Heading || TermType == WikiTermType.List) {
     const string whitespaces = " \t\r\n\v";
     if (!whitespaces.Contains(source.PreviewChar)) return null; 
   }
   var token = source.CreateToken(this.OutputTerminal);
   return token; 
 }

        protected Token CreateToken(ISourceStream source, int lastNonWhitespacePosition)
        {
            if (lastNonWhitespacePosition > source.Position)
            {
                // Remember position
                var pos = source.PreviewPosition;

                // Move position back to last non whitespace (or significant whitespace)
                source.PreviewPosition = lastNonWhitespacePosition;
                var token = source.CreateToken(this.OutputTerminal);

                // Restore position
                source.PreviewPosition = pos;
                return token;
            }
            return null;
        }

 Token MatchCodeTerminal(Terminal terminal, ParsingContext context, ISourceStream source)
 {
     var remaining = source.Text.Substring(source.Location.Position);
     var stack = 1;
     var token = "";
     while (stack > 0)
     {
         var index = remaining.IndexOf('}') + 1;
         var cut = remaining.Substring(0, index);
         token += cut;
         remaining = remaining.Substring(index);
         stack += cut.Count(x => x == '{');
         stack--;
     }
     token = token.Substring(0, token.Length - 1);
     source.PreviewPosition += token.Length;
     return source.CreateToken(terminal.OutputTerminal);
 }

 public override Token TryMatch(ParsingContext context, ISourceStream source) {
   Token result;
   if (context.VsLineScanState.Value != 0) {
     // we are continuing in line mode - restore internal env (none in this case)
     context.VsLineScanState.Value = 0;
   } else {
     //we are starting from scratch
     if (!BeginMatch(context, source)) return null;
   }
   result = CompleteMatch(context, source);
   if (result != null) return result;
   //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
   if (_isLineComment)
     return source.CreateToken(this.OutputTerminal);
   if (context.Mode == ParseMode.VsLineScan)
     return CreateIncompleteToken(context, source);
   return context.CreateErrorToken(Resources.ErrUnclosedComment);
 }

        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            while (!source.EOF())
            {
                switch (source.PreviewChar)
                {
                    case '\n':
                    case '\r':
                    case ' ':
                    case '}':
                        if (source.PreviewPosition > source.Position)
                            return source.CreateToken(this.OutputTerminal);
                        return context.CreateErrorToken(Name + " was expected");
                }
                source.PreviewPosition++;
            }

            return context.CreateErrorToken("Unbalanced " + Name);
        }

 private Token TryMatchContentSimple(ParsingContext context, ISourceStream source)
 {
     var startPos = source.PreviewPosition;
       int p = source.Text.IndexOf(_singleTerminator, startPos, Grammar.StringComparisonMode);
       if (p < 0 && IsSet(FreeTextOptions.AllowEof))
     p = source.Text.Length;
       if (p < 0)
     return source.CreateErrorToken(Resources.ErrFreeTextNoEndTag, _singleTerminator);
       var tokenText = source.Text.Substring(startPos, p - startPos);
       return source.CreateToken(this.OutputTerminal, tokenText);
 }

 public override Token TryMatch(ParsingContext context, ISourceStream source)
 {
     var Match = MatchRegex.Match(source.Text, source.PreviewPosition);
     if (!Match.Success || Match.Index != source.PreviewPosition) return null;
     source.PreviewPosition += Match.Length;
     return source.CreateToken(this.OutputTerminal);
 }