C# TokenStream.Pop方法代码示例

		private Executable ParseImport(TokenStream tokens, int indention)
		{
			tokens.SkipWhitespace();

			List<Token> fromChain = null;
			List<Token> importChain = null;
			List<Token> asChain = null;
			Token firstToken = null;

			if (tokens.PeekValue() == "from")
			{
				firstToken = tokens.Pop();
				fromChain = ParseDotChainForImport(tokens);
			}

			firstToken = firstToken ?? tokens.PopExpected("import");
			importChain = ParseDotChainForImport(tokens);

			if (tokens.PopIfPresent("as"))
			{
				asChain = ParseDotChainForImport(tokens);
				if (asChain.Count > 1) throw new ParserException(asChain[0], "Expected: variable");
			}

			return new ImportStatement(firstToken, importChain, fromChain, asChain == null ? null : asChain[0]);
		}

		public static Expression ParseLambda(TokenStream tokens)
		{
			tokens.SkipWhitespace();
			if (tokens.IsNext("lambda"))
			{
				Token lambdaToken = tokens.PopExpected("lambda");
				List<Token> argNames = new List<Token>();
				List<Expression> argValues = new List<Expression>();

				if (!tokens.PopIfPresent(":"))
				{
					bool nextAllowed = true;
					while (!tokens.PopIfPresent(":"))
					{
						if (!nextAllowed) tokens.PopExpected(":"); // throws
						Token argName = tokens.Pop();
						if (!Util.IsIdentifier(argName)) throw new ParserException(argName, "Invalid lambda arg name.");
						Expression argValue = null;
						if (tokens.PopIfPresent("="))
						{
							argValue = ExpressionParser.Parse(tokens);
						}

						argNames.Add(argName);
						argValues.Add(argValue);

						nextAllowed = tokens.PopIfPresent(",");
					}
				}
			}

			return ParseTernary(tokens);
		}

 private Executable ParseAtomicItem(TokenStream tokens, int indention)
 {
     tokens.SkipWhitespace();
     Token token = tokens.Pop();
     if (token.Value == "break") return new BreakStatement(token);
     if (token.Value == "continue") return new ContinueStatement(token);
     if (token.Value == "pass") return new PassStatement(token);
     throw new Exception("wat?");
 }

 public static Expression ParseBitshift(TokenStream tokens)
 {
     Expression left = ParseAddition(tokens);
     string next = tokens.PeekValue();
     if (BITSHIFT_OPERATORS.Contains(next))
     {
         List<Expression> expressions = new List<Expression>() { left };
         List<Token> ops = new List<Token>();
         while (BITSHIFT_OPERATORS.Contains(next))
         {
             ops.Add(tokens.Pop());
             expressions.Add(ParseAddition(tokens));
             next = tokens.PeekValue();
         }
         return new BinaryOpChain(expressions, ops);
     }
     return left;
 }

 public static Expression ParseBitwiseAnd(TokenStream tokens)
 {
     Expression left = ParseBitshift(tokens);
     string next = tokens.PeekValue();
     if (next == "^")
     {
         List<Expression> expressions = new List<Expression>() { left };
         List<Token> ops = new List<Token>();
         while (next == "^")
         {
             ops.Add(tokens.Pop());
             expressions.Add(ParseBitshift(tokens));
             next = tokens.PeekValue();
         }
         return new BinaryOpChain(expressions, ops);
     }
     return left;
 }

		public Executable Parse(TokenStream tokens, bool nestedIndentAllowed, int indention)
		{
			string next = tokens.PeekValue();

			if (nestedIndentAllowed)
			{
				switch (next)
				{
					case "def": return ParseDef(tokens, indention);
					case "class": return ParseClass(tokens, indention);
					case "if": return ParseIfLikeThing(tokens, indention);
					case "elif": return ParseIfLikeThing(tokens, indention);
					case "else": return ParseIfLikeThing(tokens, indention);
					case "for": return ParseIfLikeThing(tokens, indention);
					case "while": return ParseIfLikeThing(tokens, indention);
					default: break;
				}
			}
			switch (next)
			{
				case "break": return ParseAtomicItem(tokens, indention);
				case "continue": return ParseAtomicItem(tokens, indention);
				case "pass": return ParseAtomicItem(tokens, indention);
				case "return": return ParseReturn(tokens, indention);
				case "import": return ParseImport(tokens, indention);
				case "from": return ParseImport(tokens, indention);
				default: break;
			}


			Expression expression = ExpressionParser.Parse(tokens);

			next = tokens.PeekValue();
			if (ASSIGNMENT_OPS.Contains(next))
			{
				Token assignmentToken = tokens.Pop();
				Expression assignmentValue = ExpressionParser.Parse(tokens);
				return new Assignment(expression, assignmentToken, assignmentValue);
			}

			return new ExpressionAsExecutable(expression);
		}

		private List<Token> ParseDotChainForImport(TokenStream tokens)
		{
			List<Token> output = new List<Token>() {tokens.Pop()};
			if (output[0].Value != "*" && !Util.IsIdentifier(output[0]))
			{
				throw new ParserException(output[0], "Invalid import statement.");
			}

			while (tokens.PopIfPresent("."))
			{
				Token next = tokens.Pop();
				if (next.Value != "*" && !Util.IsIdentifier(next))
				{
					throw new ParserException(next, "Invalid import statement.");
				}
				output.Add(next);
			}

			return output;
		}

        private static Expression ParseRootEntity(TokenStream tokens)
        {
            tokens.SkipWhitespace();
            string next = tokens.PeekValue();
            if (next == "True" || next == "False")
            {
                Token token = tokens.Pop();
                return new BooleanConstant(token, next == "True");
            }
            else if (next == "None")
            {
                Token token = tokens.Pop();
                return new NullConstant(token);
            }
            else if (next.StartsWith("'") || next.StartsWith("\""))
            {
                Token token = tokens.Pop();
                int quoteSize = next.StartsWith("'''") || next.StartsWith("\"\"\"") ? 3 : 1;
                return new StringConstant(token, Util.RemoveEscapeSequences(token, next.Substring(quoteSize, next.Length - quoteSize * 2)));
            }
            else if (next.StartsWith("r'") || next.StartsWith("r\""))
            {
                Token token = tokens.Pop();
                int quoteSize = next.StartsWith("r'''") || next.StartsWith("r\"\"\"") ? 3 : 1;

                return new StringConstant(token, next.Substring(quoteSize + 1, next.Length - quoteSize * 2 - 1));
            }
            else if (next == "(")
            {
                // Tuples or parentehsis
                Token parenthesisToken = tokens.Pop();
                List<Expression> parenthesisExpressions = new List<Expression>();
                bool nextAllowed = true;
                while (!tokens.PopIfPresent(")"))
                {
                    if (!nextAllowed) tokens.PopExpected(")"); // throws
                    parenthesisExpressions.Add(Parse(tokens));
                    nextAllowed = tokens.PopIfPresent(",");
                    tokens.SkipWhitespace();
                }

                if (parenthesisExpressions.Count > 1 || nextAllowed)
                {
                    return new InlineTuple(parenthesisToken, parenthesisExpressions);
                }
                else
                {
                    return new ParenthesisGroup(parenthesisToken, parenthesisExpressions[0]);
                }
            }
            else if (next == "[")
            {
                Token bracketToken = tokens.Pop();
                List<Expression> listItems = new List<Expression>();
                bool nextAllowed = true;
                while (!tokens.PopIfPresent("]"))
                {
                    if (!nextAllowed) tokens.PopExpected("]"); // throws
                    listItems.Add(Parse(tokens));
                    nextAllowed = tokens.PopIfPresent(",");
                    tokens.SkipWhitespace();
                }
                return new InlineList(bracketToken, listItems);
            }
            else if (next == "{")
            {
                Token braceToken = tokens.Pop();
                List<Expression> dictionaryKeys = new List<Expression>();
                List<Expression> dictionaryValues = new List<Expression>();
                bool nextAllowed = true;
                while (!tokens.PopIfPresent("}"))
                {
                    if (!nextAllowed) tokens.PopExpected("}"); // throws
                    dictionaryKeys.Add(Parse(tokens));
                    tokens.PopExpected(":");
                    dictionaryValues.Add(Parse(tokens));
                    nextAllowed = tokens.PopIfPresent(",");
                    tokens.SkipWhitespace();
                }
                return new InlineDictionary(braceToken, dictionaryKeys, dictionaryValues);
            }
            else if (next.StartsWith("0x") || next.StartsWith("0o") || next.StartsWith("0X") || next.StartsWith("0O"))
            {
                Token integerToken = tokens.Pop();
                int radix = next.ToLowerInvariant().StartsWith("0x") ? 16 : 8;
                string stringValue = next.Substring(2);
                if (stringValue.Length == 0)
                {
                    throw new ParserException(integerToken, "Invalid base " + radix + " constant.");
                }
                int value = Util.ParseNumber(integerToken, stringValue, radix);
                return new IntegerConstant(integerToken, value);
            }
            else if (next[0] >= '0' && next[0] <= '9')
            {
                Token numberToken = tokens.Pop();
                if (next.Contains('.') || next.Contains('e') || next.Contains('E'))
                {
                    double value = Util.ParseDouble(numberToken);
                    return new FloatConstant(numberToken, value);
//.........这里部分代码省略.........

		private Executable ParseDef(TokenStream tokens, int currentIndent)
		{
			Token defToken = tokens.PopExpected("def");
			Token nameToken = tokens.Pop();
			if (!Util.IsIdentifier(nameToken))
			{
				throw new ParserException(nameToken, "Invalid function name.");
			}

			tokens.PopExpected("(");
			List<Token> args = new List<Token>();
			List<Expression> argValues = new List<Expression>();
			while (!tokens.PopIfPresent(")"))
			{
				if (args.Count > 0)
				{
					tokens.PopExpected(",");
				}
				Token argToken = tokens.Pop();
				if (!Util.IsIdentifier(argToken)) throw new ParserException(argToken, "Invalid argument name.");
				Expression argValue = null;
				if (tokens.PopIfPresent("="))
				{
					argValue = ExpressionParser.Parse(tokens);
				}
				args.Add(argToken);
				argValues.Add(argValue);
			}
			tokens.PopExpected(":");

			IList<Executable> body = this.ParseBlock(tokens, currentIndent, false);
			return new FunctionDefinition(defToken, nameToken, args, argValues, body);
		}

        public static Expression ParseBooleanNot(TokenStream tokens)
        {
            tokens.SkipWhitespace();
            if (tokens.IsNext("not"))
            {
                Token notToken = tokens.Pop();
                Expression expression = ParseComparisons(tokens);
                return new Negation(notToken, expression, Negation.PrefixType.BOOLEAN_NOT);
            }

            return ParseComparisons(tokens);
        }

		private Executable ParseIfLikeThing(TokenStream tokens, int currentIndention)
		{
			// This is used for if, elif, else, while, and for, because they are all similar.
			Token token = tokens.Pop();
			string tokenValue = token.Value;
			Expression condition = null;
			Token forIterator = null;
			if (tokenValue == "for")
			{
				forIterator = tokens.Pop();
				if (!Util.IsIdentifier(forIterator)) throw new ParserException(forIterator, "Expected variable name.");
				tokens.PopExpected("in");
			}
			if (tokenValue != "else")
			{
				condition = ExpressionParser.Parse(tokens);
			}
			tokens.PopExpected(":");
			IList<Executable> body = this.ParseBlock(tokens, currentIndention, false);

			if (tokenValue == "for") return new ForEachLoop(token, forIterator, condition, body);
			if (tokenValue == "while") return new WhileLoop(token, condition, body);

			return new IfRawComponent(token, condition, body);
		}

		private Executable ParseClass(TokenStream tokens, int currentIndention)
		{
			Token classToken = tokens.PopExpected("class");
			Token nameToken = tokens.Pop();
			if (!Util.IsIdentifier(nameToken)) throw new ParserException(nameToken, "Invalid class name.");
			Token baseClassName = null;
			if (tokens.PopIfPresent("("))
			{
				baseClassName = tokens.Pop();
				if (!Util.IsIdentifier(baseClassName)) throw new ParserException(baseClassName, "Invalid base class name.");
				tokens.PopExpected(")");
			}
			tokens.PopExpected(":");
			IList<Executable> members = this.ParseBlock(tokens, currentIndention, false);
			List<Executable> membersFiltered = new List<Executable>();
			foreach (Executable member in members)
			{
				if (member is PassStatement)
				{
					// this is fine.
				}
				else if (!(member is FunctionDefinition))
				{
					throw new ParserException(member.FirstToken, "Non function members of classes are not supported yet.");
				}
				else
				{
					membersFiltered.Add(member);
				}
			}

			return new ClassDefinition(classToken, nameToken, baseClassName, membersFiltered);
		}

 public static Expression ParseComparisons(TokenStream tokens)
 {
     Expression left = ParseBitwiseOr(tokens);
     string next = tokens.PeekValue();
     if (COMPARISON_TOKENS.Contains(next))
     {
         List<Expression> expressions = new List<Expression>() { left };
         List<Token> comparisons = new List<Token>();
         while (COMPARISON_TOKENS.Contains(next))
         {
             comparisons.Add(tokens.Pop());
             expressions.Add(ParseBitwiseOr(tokens));
             next = tokens.PeekValue();
         }
         return new ComparisonChain(expressions, comparisons);
     }
     return left;
 }

 public static Expression ParseMultiplication(TokenStream tokens)
 {
     Expression left = ParseNot(tokens);
     string next = tokens.PeekValue();
     if (MULTIPLICATION_OPERATORS.Contains(next))
     {
         List<Expression> expressions = new List<Expression>() { left };
         List<Token> ops = new List<Token>();
         while (MULTIPLICATION_OPERATORS.Contains(next))
         {
             ops.Add(tokens.Pop());
             expressions.Add(ParseNot(tokens));
             next = tokens.PeekValue();
         }
         return new BinaryOpChain(expressions, ops);
     }
     return left;
 }

        private static Expression ParseEntityWithSuffix(TokenStream tokens)
        {
            Expression expression = ParseRootEntity(tokens);
            bool nextAllowed;
            bool keepGoing = true;
            string next = tokens.PeekValue();

            while (keepGoing)
            {
                switch (next)
                {
                    case "[":
                        // indexing or slicing
                        Token bracketToken = tokens.Pop();
                        List<Expression> sliceComponents = new List<Expression>();
                        nextAllowed = true;
                        while (!tokens.PopIfPresent("]"))
                        {
                            if (!nextAllowed) tokens.PopExpected("]"); // throws

                            if (tokens.IsNext(":"))
                            {
                                sliceComponents.Add(null);
                            }
                            else
                            {
                                sliceComponents.Add(Parse(tokens));
                            }
                            nextAllowed = tokens.PopIfPresent(":");
                        }
                        if (nextAllowed)
                        {
                            sliceComponents.Add(null);
                        }
                        if (sliceComponents.Count == 0)
                        {
                            throw new ParserException(bracketToken, "Unexpected token.");
                        }
                        else if (sliceComponents.Count == 1)
                        {
                            expression = new IndexExpression(expression, bracketToken, sliceComponents[0]);
                        }
                        else if (sliceComponents.Count <= 3)
                        {
                            expression = new SliceExpression(expression, bracketToken, sliceComponents);
                        }
                        else
                        {
                            throw new ParserException(bracketToken, "Slice expression has too many components.");
                        }
                        break;
                    case "(":
                        // function call
                        Token openParen = tokens.Pop();
                        nextAllowed = true;
                        List<Expression> args = new List<Expression>();
                        while (!tokens.PopIfPresent(")"))
                        {
                            if (!nextAllowed) tokens.PopExpected(")"); // throws
                            args.Add(Parse(tokens));
                            nextAllowed = tokens.PopIfPresent(",");
                        }

                        expression = new FunctionInvocation(expression, args);
                        break;
                    case ".":
                        // dot field
                        Token dotToken = tokens.Pop();
                        Token fieldToken = tokens.Pop();
                        if (!Util.IsIdentifier(fieldToken))
                        {
                            throw new ParserException(fieldToken, "Invalid field.");
                        }
                        expression = new DotField(expression, dotToken, fieldToken, fieldToken.Value);
                        break;
                    default:
                        keepGoing = false;
                        break;
                }
                next = tokens.PeekValue();
            }

            return expression;
        }