C# TokenStream.SkipWhitespace方法代码示例

		public static Expression ParseLambda(TokenStream tokens)
		{
			tokens.SkipWhitespace();
			if (tokens.IsNext("lambda"))
			{
				Token lambdaToken = tokens.PopExpected("lambda");
				List<Token> argNames = new List<Token>();
				List<Expression> argValues = new List<Expression>();

				if (!tokens.PopIfPresent(":"))
				{
					bool nextAllowed = true;
					while (!tokens.PopIfPresent(":"))
					{
						if (!nextAllowed) tokens.PopExpected(":"); // throws
						Token argName = tokens.Pop();
						if (!Util.IsIdentifier(argName)) throw new ParserException(argName, "Invalid lambda arg name.");
						Expression argValue = null;
						if (tokens.PopIfPresent("="))
						{
							argValue = ExpressionParser.Parse(tokens);
						}

						argNames.Add(argName);
						argValues.Add(argValue);

						nextAllowed = tokens.PopIfPresent(",");
					}
				}
			}

			return ParseTernary(tokens);
		}

		private Executable ParseImport(TokenStream tokens, int indention)
		{
			tokens.SkipWhitespace();

			List<Token> fromChain = null;
			List<Token> importChain = null;
			List<Token> asChain = null;
			Token firstToken = null;

			if (tokens.PeekValue() == "from")
			{
				firstToken = tokens.Pop();
				fromChain = ParseDotChainForImport(tokens);
			}

			firstToken = firstToken ?? tokens.PopExpected("import");
			importChain = ParseDotChainForImport(tokens);

			if (tokens.PopIfPresent("as"))
			{
				asChain = ParseDotChainForImport(tokens);
				if (asChain.Count > 1) throw new ParserException(asChain[0], "Expected: variable");
			}

			return new ImportStatement(firstToken, importChain, fromChain, asChain == null ? null : asChain[0]);
		}

 private Executable ParseAtomicItem(TokenStream tokens, int indention)
 {
     tokens.SkipWhitespace();
     Token token = tokens.Pop();
     if (token.Value == "break") return new BreakStatement(token);
     if (token.Value == "continue") return new ContinueStatement(token);
     if (token.Value == "pass") return new PassStatement(token);
     throw new Exception("wat?");
 }

		public static Expression ParseBooleanAnd(TokenStream tokens)
		{
			tokens.SkipWhitespace();
			Expression left = ParseBooleanNot(tokens);
			if (tokens.PopIfPresent("and"))
			{
				Expression right = ParseBooleanAnd(tokens);
				return new BooleanCombinator(left, right, "and");
			}
			return left;
		}

		// YOU LEFT OFF HERE
		// You were about to go through and document exactly what each indention variable meant more clearly.
		private IList<Executable> ParseBlock(TokenStream tokens, int ownerIndention, bool start)
		{
			int blockIndention = tokens.PeekIndention();
			if (start)
			{
				if (blockIndention != 0)
				{
					throw new ParserException(tokens.Peek(), "Unexpected indention");
				}
			}
			else
			{
				if (blockIndention == -1)
				{
					// This indicates the code is on the same line. Parse one line.
					// def foo(): return 42
					Executable exec = this.Parse(tokens, false, -1);
					return new List<Executable>() { exec };
				}

				if (blockIndention <= ownerIndention)
				{
					// No indention was found. But it is required.
					throw new ParserException(tokens.Peek(), "Expected: indention");
				}
			}

			int requiredIndention = blockIndention;
			
			List<Executable> code = new List<Executable>();
			while (tokens.HasMore)
			{
				int currentIndention = tokens.PeekIndention();

				// any new indention should be handled by a recursive call
				if (currentIndention > requiredIndention) throw new ParserException(tokens.Peek(), "Unexpected indention");

				// if it's indented less than the required but more than the previous, then that's not right
				// e.g.
				// def foo()
				//     x = 1
				//   return x # this is wrong
				if (currentIndention < requiredIndention && currentIndention > ownerIndention) throw new ParserException(tokens.Peek(), "Unexpected indention");

				// def foo()
				//     x = 42
				// y = 3.14 # this is no longer part of foo()
				// start is excluded because when start is true, the owner indention and current indention are the same.
				if (!start && currentIndention <= ownerIndention) return code;

				tokens.SkipWhitespace();
				if (tokens.HasMore)
				{
					code.Add(this.Parse(tokens, true, currentIndention));
				}
				else
				{
					return code;
				}
			}
			return code;
		}

		private Executable ParseReturn(TokenStream tokens, int indention)
		{
			tokens.SkipWhitespace();
			Token token = tokens.PopExpected("return");
			Expression value = null;
			if (tokens.HasMore && tokens.PeekType() != TokenType.NEWLINE)
			{
				value = ExpressionParser.Parse(tokens);
			}
			return new ReturnStatement(token, value);
		}

        private static Expression ParseRootEntity(TokenStream tokens)
        {
            tokens.SkipWhitespace();
            string next = tokens.PeekValue();
            if (next == "True" || next == "False")
            {
                Token token = tokens.Pop();
                return new BooleanConstant(token, next == "True");
            }
            else if (next == "None")
            {
                Token token = tokens.Pop();
                return new NullConstant(token);
            }
            else if (next.StartsWith("'") || next.StartsWith("\""))
            {
                Token token = tokens.Pop();
                int quoteSize = next.StartsWith("'''") || next.StartsWith("\"\"\"") ? 3 : 1;
                return new StringConstant(token, Util.RemoveEscapeSequences(token, next.Substring(quoteSize, next.Length - quoteSize * 2)));
            }
            else if (next.StartsWith("r'") || next.StartsWith("r\""))
            {
                Token token = tokens.Pop();
                int quoteSize = next.StartsWith("r'''") || next.StartsWith("r\"\"\"") ? 3 : 1;

                return new StringConstant(token, next.Substring(quoteSize + 1, next.Length - quoteSize * 2 - 1));
            }
            else if (next == "(")
            {
                // Tuples or parentehsis
                Token parenthesisToken = tokens.Pop();
                List<Expression> parenthesisExpressions = new List<Expression>();
                bool nextAllowed = true;
                while (!tokens.PopIfPresent(")"))
                {
                    if (!nextAllowed) tokens.PopExpected(")"); // throws
                    parenthesisExpressions.Add(Parse(tokens));
                    nextAllowed = tokens.PopIfPresent(",");
                    tokens.SkipWhitespace();
                }

                if (parenthesisExpressions.Count > 1 || nextAllowed)
                {
                    return new InlineTuple(parenthesisToken, parenthesisExpressions);
                }
                else
                {
                    return new ParenthesisGroup(parenthesisToken, parenthesisExpressions[0]);
                }
            }
            else if (next == "[")
            {
                Token bracketToken = tokens.Pop();
                List<Expression> listItems = new List<Expression>();
                bool nextAllowed = true;
                while (!tokens.PopIfPresent("]"))
                {
                    if (!nextAllowed) tokens.PopExpected("]"); // throws
                    listItems.Add(Parse(tokens));
                    nextAllowed = tokens.PopIfPresent(",");
                    tokens.SkipWhitespace();
                }
                return new InlineList(bracketToken, listItems);
            }
            else if (next == "{")
            {
                Token braceToken = tokens.Pop();
                List<Expression> dictionaryKeys = new List<Expression>();
                List<Expression> dictionaryValues = new List<Expression>();
                bool nextAllowed = true;
                while (!tokens.PopIfPresent("}"))
                {
                    if (!nextAllowed) tokens.PopExpected("}"); // throws
                    dictionaryKeys.Add(Parse(tokens));
                    tokens.PopExpected(":");
                    dictionaryValues.Add(Parse(tokens));
                    nextAllowed = tokens.PopIfPresent(",");
                    tokens.SkipWhitespace();
                }
                return new InlineDictionary(braceToken, dictionaryKeys, dictionaryValues);
            }
            else if (next.StartsWith("0x") || next.StartsWith("0o") || next.StartsWith("0X") || next.StartsWith("0O"))
            {
                Token integerToken = tokens.Pop();
                int radix = next.ToLowerInvariant().StartsWith("0x") ? 16 : 8;
                string stringValue = next.Substring(2);
                if (stringValue.Length == 0)
                {
                    throw new ParserException(integerToken, "Invalid base " + radix + " constant.");
                }
                int value = Util.ParseNumber(integerToken, stringValue, radix);
                return new IntegerConstant(integerToken, value);
            }
            else if (next[0] >= '0' && next[0] <= '9')
            {
                Token numberToken = tokens.Pop();
                if (next.Contains('.') || next.Contains('e') || next.Contains('E'))
                {
                    double value = Util.ParseDouble(numberToken);
                    return new FloatConstant(numberToken, value);
//.........这里部分代码省略.........

        private static Expression ParseNot(TokenStream tokens)
        {
            tokens.SkipWhitespace();
            string next = tokens.PeekValue();
            if (next == "+" || next == "-" || next == "~")
            {
                Token token = tokens.Pop();
                Expression root = ParseExponents(tokens);
                return new Negation(token, root,
                    token.Value == "+"
                        ? Negation.PrefixType.POSITIVE
                        : token.Value == "-"
                            ? Negation.PrefixType.NEGATIVE
                            : Negation.PrefixType.BITWISE_NOT);
            }

            return ParseExponents(tokens);
        }

        public static Expression ParseBooleanNot(TokenStream tokens)
        {
            tokens.SkipWhitespace();
            if (tokens.IsNext("not"))
            {
                Token notToken = tokens.Pop();
                Expression expression = ParseComparisons(tokens);
                return new Negation(notToken, expression, Negation.PrefixType.BOOLEAN_NOT);
            }

            return ParseComparisons(tokens);
        }