C# ISourceStream.EOF方法代码示例

    private Token CompleteMatch(ISourceStream source) {
      if (source.EOF())
        return null;

      do {
        // Match NewLine
        var lookAhead = source.PreviewChar;
        if (LineTerminators.IndexOf(lookAhead) >= 0)
        {
          source.PreviewPosition++;
          // Treat \r\n as single NewLine
          if (!source.EOF() && lookAhead == '\r' && source.PreviewChar == '\n')
            source.PreviewPosition++;
          break;
        }

        // Eat up whitespace
        if (GrammarData.Grammar.WhitespaceChars.IndexOf(lookAhead) >= 0)
        {
          source.PreviewPosition++;
          continue;
        }

        // Fail on anything else
        return null;
      }
      while (!source.EOF());

      // Create output token
      return source.CreateToken(this.OutputTerminal);
    }

 private Token CompleteMatch(ParsingContext context, ISourceStream source) {
   //Find end symbol
   while (!source.EOF()) {
     int firstCharPos;
     if (EndSymbols.Count == 1)
       firstCharPos = source.Text.IndexOf(EndSymbols[0], source.PreviewPosition);
     else 
       firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.PreviewPosition);
     if (firstCharPos < 0) {
       source.PreviewPosition = source.Text.Length;
       return null; //indicating error
     }
     //We found a character that might start an end symbol; let's see if it is true.
     source.PreviewPosition = firstCharPos;
     foreach (string endSymbol in EndSymbols) {
       if (source.MatchSymbol(endSymbol)) {
         //We found end symbol; eat end symbol only if it is not line comment.
         // For line comment, leave LF symbol there, it might be important to have a separate LF token
         if (!_isLineComment)
           source.PreviewPosition += endSymbol.Length;
         return source.CreateToken(this.OutputTerminal); 
       }//if
     }//foreach endSymbol
     source.PreviewPosition++; //move to the next char and try again    
   }//while
   return null; //might happen if we found a start char of end symbol, but not the full endSymbol
 }//method

    public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
    {
      if (!source.MatchSymbol(_startSymbol, false)) return null;
      source.Position += _startSymbol.Length;

      while (!source.EOF())
      {
        int firstCharPos = source.Text.IndexOf(_endSymbol, source.Position);

        if (firstCharPos < 0)
        {
          source.Position = source.Text.Length;

          if (_isLineComment)
            return TokenAst.Create(this, context, source.TokenStart, source.GetLexeme());
          else
            return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block");
        }
      
        source.Position = firstCharPos;
        if (source.MatchSymbol(_endSymbol, false))
        {
          source.Position += _endSymbol.Length;
          return TokenAst.Create(this, context, source.TokenStart, source.GetLexeme());
        }
        
        source.Position++; 
      }

      throw new NotSupportedException();
    }

        public override Token TryMatch(CompilerContext context, ISourceStream source)
        {
            bool ignoreCase = !Grammar.CaseSensitive;
              //Check starting symbol
              if (!source.MatchSymbol(StartSymbol, ignoreCase)) return null;
              //Find end symbol
              source.Position += StartSymbol.Length;

              while(!source.EOF()) {
            int firstCharPos;
            if (EndSymbols.Count == 1)
              firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position);
            else
              firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position);
            if (firstCharPos < 0) {
              source.Position = source.Text.Length;
              if (_isLineComment) //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
            return Token.Create(this, context, source.TokenStart, source.GetLexeme());
              else
            return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block");
            }
            //We found a character that might start an end symbol; let's see if it is true.
            source.Position = firstCharPos;
            foreach (string endSymbol in EndSymbols)
              if (source.MatchSymbol(endSymbol, ignoreCase)) {
            //We found end symbol
            source.Position += endSymbol.Length;
            return Token.Create(this, context, source.TokenStart, source.GetLexeme());
              }//if
            source.Position++; //move to the next char and try again
              }//while
              return null; //never happens
        }

        private Token MatchQuoted(ParsingContext context, ISourceStream source)
        {
            char quoteChar = source.PreviewChar;
            if ((quoteChar != '\'') && (quoteChar != '"'))
            {
                return null;
            }

            source.PreviewPosition++;
            while (!source.EOF())
            {
                if (source.PreviewChar == quoteChar)
                {
                    source.PreviewPosition++;
                    return source.CreateToken(this.OutputTerminal);
                }

                // Escaped?
                if (source.PreviewChar == '\\')
                {
                    // Consume next
                    ++source.PreviewPosition;
                }

                source.PreviewPosition++;
            }

            return context.CreateErrorToken("Unbalanced quoted string");
        }

    public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
    {
      bool isVerbatim = false;
      int start = source.Position;

      if (source.CurrentChar == '@')
      {
        isVerbatim = true;
        source.Position++;
        start++;
      }

      if (IsCurrentQuote(source))
      {
        source.Position++;
        start++;
      }
      else 
        return null;

      while (!source.EOF())
      {
        if (!isVerbatim)
        {
          if (source.CurrentChar == '\\')
          {
            //TODO: Escape processing
            source.Position += 2;
            continue;
          }
          else
            //Single line string ends incorrectly
            if (ParserData.LineTerminators.IndexOf(source.CurrentChar) >= 0)
              return null;
        }

        if (IsCurrentQuote(source)) break;
        
        source.Position++;
      }

      if (IsCurrentQuote(source))
        source.Position++;
      else
        return null;
      
      string lexeme = source.GetLexeme();
      string body = source.Text.Substring(start, source.Position - start - 1);
      //TODO: handle this in escape processing
      if (!isVerbatim)
        body = body.Replace("\\'", "'").Replace("\\\"", "\"").Replace("\\\\", "\\");

      TokenAst token = TokenAst.Create(this, context, source.TokenStart, lexeme, body);     
      return token;

      //return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Failed to convert the value");
    }

        public override Token TryMatch(ParsingContext context, ISourceStream source)
        {
            while (!source.EOF())
            {
                switch (source.PreviewChar)
                {
                    case '\n':
                    case '\r':
                    case ' ':
                    case '}':
                        if (source.PreviewPosition > source.Position)
                            return source.CreateToken(this.OutputTerminal);
                        return context.CreateErrorToken(Name + " was expected");
                }
                source.PreviewPosition++;
            }

            return context.CreateErrorToken("Unbalanced " + Name);
        }

        private Token CompleteMatch(ParsingContext context, ISourceStream source, byte commentLevel)
        {
            if (commentLevel == 0)
            {
                var line_breaks = new char[] { '\n', '\r', '\v' };
                var firstCharPos = source.Text.IndexOfAny(line_breaks, source.PreviewPosition);
                if (firstCharPos > 0)
                {
                    source.PreviewPosition = firstCharPos;
                }
                else
                {
                    source.PreviewPosition = source.Text.Length;
                }

                return source.CreateToken(this.OutputTerminal);
            }

            while (!source.EOF())
            {      
               string text = source.Text.Substring(source.PreviewPosition);
                var matches = Regex.Matches(text, @"\](=*)\]");
                foreach (Match match in matches)
                {
                    if (match.Groups[1].Value.Length == (int)commentLevel - 1)
                    {
                        source.PreviewPosition += match.Index + match.Length;

                        if (context.VsLineScanState.Value != 0)
                        {
                            SourceLocation tokenStart = new SourceLocation();
                            tokenStart.Position = 0;

                            string lexeme = source.Text.Substring(0, source.PreviewPosition);

                            context.VsLineScanState.Value = 0;
                            return new Token(this, tokenStart, lexeme, null);
                        }
                        else
                        {
                            return source.CreateToken(this.OutputTerminal);
                        }
                    }
                }

                source.PreviewPosition++;
            }
            
			context.VsLineScanState.TokenSubType = commentLevel;
            return null;
        }

 /// <summary>Skips whitespace characters in the input stream. </summary>
 /// <remarks>Override this method if your language has non-standard whitespace characters.</remarks>
 /// <param name="source">Source stream.</param>
 public virtual void SkipWhitespace(ISourceStream source)
 {
     while (!source.EOF()) {
     switch (source.PreviewChar) {
       case ' ':  case '\t':
     break;
       case '\r':   case '\n':  case '\v':
     if (UsesNewLine) return; //do not treat as whitespace if language is line-based
     break;
       default:
     return;
     }//switch
     source.PreviewPosition++;
       }//while
 }

 private void MoveSourcePositionAfterTerminator(ISourceStream source) {
   while(!source.EOF()) {
     while(source.PreviewChar != Terminator[0])
       source.PreviewPosition++;
     if(source.MatchSymbol(Terminator, !Grammar.CaseSensitive)) {
       source.PreviewPosition += Terminator.Length;
       return;
     }//if
   }//while
 }//method

        protected override bool ReadBody(ISourceStream source, ScanDetails details)
        {
            //remember start - it may be different from source.TokenStart, we may have skipped
              int start = source.Position;
              //Figure out digits set
              string digits = GetDigits(details);
              bool isDecimal = !details.IsSet(ScanFlags.NonDecimal);
              bool allowFloat = !IsSet(TermOptions.NumberIntOnly);

              while (!source.EOF()) {
            char current = source.CurrentChar;
            //1. If it is a digit, just continue going
            if (digits.IndexOf(current) >= 0) {
              source.Position++;
              continue;
            }
            //2. Check if it is a dot
            if (current == DecimalSeparator && allowFloat) {
              //If we had seen already a dot or exponent, don't accept this one;
              //In python number literals (NumberAllowPointFloat) a point can be the first and last character,
              //otherwise we accept dot only if it is followed by a digit
              if (details.IsSet(ScanFlags.HasDotOrExp) || (digits.IndexOf(source.NextChar) < 0) && !IsSet(TermOptions.NumberAllowStartEndDot))
            break; //from while loop
              details.Flags |= ScanFlags.HasDot;
              source.Position++;
              continue;
            }
            //3. Only for decimals - check if it is (the first) exponent symbol
            if (allowFloat && isDecimal && (details.ControlSymbol == null) && (ExponentSymbols.IndexOf(current) >= 0)) {
              char next = source.NextChar;
              bool nextIsSign = next == '-' || next == '+';
              bool nextIsDigit = digits.IndexOf(next) >= 0;
              if (!nextIsSign && !nextIsDigit)
            break;  //Exponent should be followed by either sign or digit
              //ok, we've got real exponent
              details.ControlSymbol = current.ToString(); //remember the exp char
              details.Flags |= ScanFlags.HasExp;
              source.Position++;
              if (nextIsSign)
            source.Position++; //skip +/- explicitly so we don't have to deal with them on the next iteration
              continue;
            }
            //4. It is something else (not digit, not dot or exponent) - we're done
            break; //from while loop
              }//while
              int end = source.Position;
              details.Body = source.Text.Substring(start, end - start);
              return true;
        }

        private Token MatchUnquoted(ParsingContext context, ISourceStream source)
        {
            if (source.PreviewChar == '{')
            {
                // Member names can't start with {
                if (this.IsMemberName)
                {
                    return null;
                }

                // Check for special {} at start of token indicating that this is a STRING token.
                if (source.NextPreviewChar != '}')
                {
                    return null;
                }

                source.PreviewPosition += 2;
            }

            var runningBraceTotal = 0;

            // This variable tracks the position of the last non whitespace (or significant whitespace).
            var lastNonWhitespacePosition = source.PreviewPosition;
            while (!source.EOF())
            {
                bool isWhiteSpace = false;
                switch (source.PreviewChar)
                {
                    case '{':
                        runningBraceTotal++;
                        break;

                    case '}':
                        if (--runningBraceTotal < 0)
                        {
                            return this.CreateToken(source, lastNonWhitespacePosition);
                        }

                        break;

                    case ',':
                        if (runningBraceTotal == 0)
                        {
                            return this.CreateToken(source, lastNonWhitespacePosition);
                        }

                        break;

                    case '=':
                        if (runningBraceTotal == 0)
                        {
                            // Equal sign. Only allowed after MemberNames.
                            return this.IsMemberName
                                ? this.CreateToken(source, lastNonWhitespacePosition)
                                : null;
                        }

                        break;

                    case '\\':
                        source.PreviewPosition++;
                        break;

                    default:
                        isWhiteSpace = Char.IsWhiteSpace(source.PreviewChar);
                        break;
                }

                source.PreviewPosition++;

                if (!isWhiteSpace)
                {
                    lastNonWhitespacePosition = source.PreviewPosition;
                }
            }

            return context.CreateErrorToken("Unterminated string terminal");
        }

    protected override bool ReadBody(ISourceStream source, CompoundTokenDetails details) {
      //remember start - it may be different from source.TokenStart, we may have skipped prefix
      int start = source.Position;
      char current = source.CurrentChar;
      if (current == '-' || current == '+') {
        details.Sign = current.ToString();
        source.Position++;
      }
      //Figure out digits set
      string digits = GetDigits(details);
      bool isDecimal = !details.IsSet((short) (NumberFlags.Binary | NumberFlags.Octal | NumberFlags.Hex));
      bool allowFloat = !IsSet(NumberFlags.IntOnly);
      bool foundDigits = false;

      while (!source.EOF()) {
        current = source.CurrentChar;
        //1. If it is a digit, just continue going
        if (digits.IndexOf(current) >= 0) {
          source.Position++;
          foundDigits = true; 
          continue;
        }
        //2. Check if it is a dot in float number
        bool isDot = current == DecimalSeparator;
        if (allowFloat && isDot) {
          //If we had seen already a dot or exponent, don't accept this one;
          bool hasDotOrExp = details.IsSet((short) (NumberFlags.HasDot | NumberFlags.HasExp));
          if (hasDotOrExp) break; //from while loop
          //In python number literals (NumberAllowPointFloat) a point can be the first and last character,
          //We accept dot only if it is followed by a digit
          if (digits.IndexOf(source.NextChar) < 0 && !IsSet(NumberFlags.AllowStartEndDot))
            break; //from while loop
          details.Flags |= (int) NumberFlags.HasDot;
          source.Position++;
          continue;
        }
        //3. Check if it is int number followed by dot or exp symbol
        bool isExpSymbol = (details.ExponentSymbol == null) && ExponentSymbols.IndexOf(current) >= 0;
        if (!allowFloat && foundDigits && (isDot || isExpSymbol)) {
          //If no partial float allowed then return false - it is not integer, let float terminal recognize it as float
          if (IsSet(NumberFlags.AvoidPartialFloat)) return false;  
          //otherwise break, it is integer and we're done reading digits
          break;
        }


        //4. Only for decimals - check if it is (the first) exponent symbol
        if (allowFloat && isDecimal && isExpSymbol) {
          char next = source.NextChar;
          bool nextIsSign = next == '-' || next == '+';
          bool nextIsDigit = digits.IndexOf(next) >= 0;
          if (!nextIsSign && !nextIsDigit)
            break;  //Exponent should be followed by either sign or digit
          //ok, we've got real exponent
          details.ExponentSymbol = current.ToString(); //remember the exp char
          details.Flags |= (int) NumberFlags.HasExp;
          source.Position++;
          if (nextIsSign)
            source.Position++; //skip +/- explicitly so we don't have to deal with them on the next iteration
          continue;
        }
        //4. It is something else (not digit, not dot or exponent) - we're done
        break; //from while loop
      }//while
      int end = source.Position;
      if (!foundDigits) 
        return false; 
      details.Body = source.Text.Substring(start, end - start);
      return true;
    }

        public override void SkipWhitespace(ISourceStream source)
        {
            while (!source.EOF())
            {
                switch (source.PreviewChar)
                {
                    case '\r':
                    case '\v':
                    case ' ':
                    case '\t':
                        break;

                    case '\n':
                        if (source.NextPreviewChar != ' ' &&
                            source.NextPreviewChar != '\t' &&
                            source.NextPreviewChar != '\n' &&
                            source.NextPreviewChar != '\r' &&
                            source.NextPreviewChar != '\v' &&
                            source.NextPreviewChar != ';')
                            return;
                        break;

                    default:
                        return;
                }

                source.PreviewPosition++;
            }
        }

        public override void SkipWhitespace(ISourceStream source)
        {
            while (!source.EOF())
            {
                int count = SkipSingleWhitespace(source);

                if (count == 0) return;

                else source.PreviewPosition += count;
            }
        }