C# ISourceStream.GetLexeme方法代码示例

    public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
    {
      if (!source.MatchSymbol(_startSymbol, false)) return null;
      source.Position += _startSymbol.Length;

      while (!source.EOF())
      {
        int firstCharPos = source.Text.IndexOf(_endSymbol, source.Position);

        if (firstCharPos < 0)
        {
          source.Position = source.Text.Length;

          if (_isLineComment)
            return TokenAst.Create(this, context, source.TokenStart, source.GetLexeme());
          else
            return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block");
        }
      
        source.Position = firstCharPos;
        if (source.MatchSymbol(_endSymbol, false))
        {
          source.Position += _endSymbol.Length;
          return TokenAst.Create(this, context, source.TokenStart, source.GetLexeme());
        }
        
        source.Position++; 
      }

      throw new NotSupportedException();
    }

        public override Token TryMatch(CompilerContext context, ISourceStream source)
        {
            bool ignoreCase = !Grammar.CaseSensitive;
              //Check starting symbol
              if (!source.MatchSymbol(StartSymbol, ignoreCase)) return null;
              //Find end symbol
              source.Position += StartSymbol.Length;

              while(!source.EOF()) {
            int firstCharPos;
            if (EndSymbols.Count == 1)
              firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position);
            else
              firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position);
            if (firstCharPos < 0) {
              source.Position = source.Text.Length;
              if (_isLineComment) //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
            return Token.Create(this, context, source.TokenStart, source.GetLexeme());
              else
            return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Unclosed comment block");
            }
            //We found a character that might start an end symbol; let's see if it is true.
            source.Position = firstCharPos;
            foreach (string endSymbol in EndSymbols)
              if (source.MatchSymbol(endSymbol, ignoreCase)) {
            //We found end symbol
            source.Position += endSymbol.Length;
            return Token.Create(this, context, source.TokenStart, source.GetLexeme());
              }//if
            source.Position++; //move to the next char and try again
              }//while
              return null; //never happens
        }

 public override Token TryMatch(CompilerContext context, ISourceStream source) {
   Match m = _expression.Match(source.Text, source.Position);
   if (!m.Success || m.Index != source.Position) 
     return null;
   source.Position += m.Length;
   string text = source.GetLexeme();
   return new Token(this, source.TokenStart, text, null);
 }

 public override Token TryMatch(CompilerContext context, ISourceStream source)
 {
     Match m = _expression.Match(source.Text, source.Position);
       if (!m.Success)
     return null;
       source.Position += m.Length + 1;
       string text = source.GetLexeme();
       return Token.Create(this, context, source.TokenStart, text);
 }

/*
    private static List<string> _firsts = new List<string>() { "'", "\"", "@" };
*/
    #endregion

    #region Init

    public override TokenAst TryMatch(CompilerContext context, ISourceStream source)
    {
      bool isVerbatim = false;
      int start = source.Position;

      if (source.CurrentChar == '@')
      {
        isVerbatim = true;
        source.Position++;
        start++;
      }

      if (IsCurrentQuote(source))
      {
        source.Position++;
        start++;
      }
      else 
        return null;

      while (!source.EOF())
      {
        if (!isVerbatim)
        {
          if (source.CurrentChar == '\\')
          {
            //TODO: Escape processing
            source.Position += 2;
            continue;
          }
          if (LRParser.LineTerminators.IndexOf(source.CurrentChar) >= 0)
            return null;
        }

        if (IsCurrentQuote(source)) break;
        
        source.Position++;
      }

      if (IsCurrentQuote(source))
        source.Position++;
      else
        return null;
      
      string lexeme = source.GetLexeme();
      string body = source.Text.Substring(start, source.Position - start - 1);
      //TODO: handle this in escape processing
      if (!isVerbatim)
        body = body.Replace("\\'", "'").Replace("\\\"", "\"").Replace("\\\\", "\\");

      TokenAst token = TokenAst.Create(this, context, source.TokenStart, lexeme, body);     
      return token;

      //return Grammar.CreateSyntaxErrorToken(context, source.TokenStart, "Failed to convert the value");
    }

 public override Token TryMatch(CompilerContext context, ISourceStream source) {
   char current = source.CurrentChar;
   if (!LineTerminators.Contains(current)) return null;
   //Treat \r\n as a single terminator
   bool doExtraShift = (current == '\r' && source.NextChar == '\n');
   source.Position++; //main shift
   if (doExtraShift)
     source.Position++;
   Token result = new Token(this, source.TokenStart, source.GetLexeme(), null);
   return result;
 }

 public override Token TryMatch(CompilerContext context, ISourceStream source)
 {
     Token result;
       if (context.ScannerState.Value != 0) {
     // we are continuing in line mode - restore internal env (none in this case)
     context.ScannerState.Value = 0;
       } else {
     //we are starting from scratch
     if (!BeginMatch(context, source)) return null;
       }
       result = CompleteMatch(context, source);
       if (result != null) return result;
       //if it is LineComment, it is ok to hit EOF without final line-break; just return all until end.
       if (_isLineComment)
     return new Token(this, source.TokenStart, source.GetLexeme(), null);
       if (context.Mode == CompileMode.VsLineScan)
     return CreateIncompleteToken(context, source);
       return context.CreateErrorTokenAndReportError(source.TokenStart, string.Empty, "Unclosed comment block");
 }

 private Token CreateIncompleteToken(CompilerContext context, ISourceStream source)
 {
     source.Position = source.Text.Length;
       Token result = Token.Create(this, context, source.TokenStart, source.GetLexeme());
       result.Flags |= AstNodeFlags.IsIncomplete;
       context.ScannerState.TokenKind = this.MultilineKind;
       return result;
 }

 private Token CompleteMatch(CompilerContext context, ISourceStream source)
 {
     //Find end symbol
       while (!source.EOF()) {
     int firstCharPos;
     if (EndSymbols.Count == 1)
       firstCharPos = source.Text.IndexOf(EndSymbols[0], source.Position);
     else
       firstCharPos = source.Text.IndexOfAny(_endSymbolsFirsts, source.Position);
     if (firstCharPos < 0) {
       source.Position = source.Text.Length;
       return null; //indicating error
     }
     //We found a character that might start an end symbol; let's see if it is true.
     source.Position = firstCharPos;
     foreach (String endSymbol in EndSymbols) {
       if (source.MatchSymbol(endSymbol, !Grammar.CaseSensitive)) {
     //We found end symbol; eat end symbol only if it is not line comment.
     // For line comment, leave LF symbol there, it might be important to have a separate LF token
     if (!_isLineComment)
       source.Position += endSymbol.Length;
     return Token.Create(this, context, source.TokenStart, source.GetLexeme());
       }//if
     }//foreach endSymbol
     source.Position++; //move to the next char and try again
       }//while
       return null; //might happen if we found a start char of end symbol, but not the full endSymbol
 }

 protected virtual TokenAst CreateToken(CompilerContext context, ISourceStream source)
 {
   string lexeme = source.GetLexeme();
   TokenAst token = TokenAst.Create(this, context, source.TokenStart, lexeme, lexeme);
   return token;
 }

 private Token CreateIncompleteToken(CompilerContext context, ISourceStream source)
 {
     source.Position = source.Text.Length;
       Token result = new Token(this, source.TokenStart, source.GetLexeme(), null);
       result.Flags |= TokenFlags.IsIncomplete;
       context.ScannerState.TerminalIndex = this.MultilineIndex;
       return result;
 }