C# ITokenizer.Read方法代码示例

        /// <summary>
        /// Reads a while statement from the input.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="prev">The token to append what is read into.</param>
        /// <returns>The object that was read.</returns>
        protected virtual IParseStatement ReadWhile(ITokenizer input, ref Token prev)
        {
            var debug = input.Read(); // read 'while'
            WhileItem w = new WhileItem();
            if (debug.Value != "while")
                throw new InvalidOperationException(string.Format(Resources.MustBeOn, "while", "ReadWhile"));
            
            // read the expression
            w.Exp = ReadExp(input, ref debug);

            // read 'do'
            var name = Read(input, ref debug);
            if (name.Value != "do")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, name.Value, "while", "do"),
                    input.Name, name);

            // read the block
            w.Block = ReadBlock(input, ref debug);

            // read 'end'
            name = Read(input, ref debug);
            if (name.Value != "end")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, name.Value, "while", "end"),
                    input.Name, name);

            prev.Append(debug);
            w.Debug = debug;
            return w;
        }

        /// <summary>
        /// Reads a prefix-expression from the input.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="token">The token to append the total token onto.</param>
        /// <returns>The expression that was read.</returns>
        protected virtual IParseExp ReadPrefixExp(ITokenizer input, ref Token token)
        {
            Stack<UnaryInfo> ex = new Stack<UnaryInfo>();
            IParseExp o = null;
            Token last, debug = input.Peek();
            debug.Value = "";

            // check for unary operators
            last = input.Peek();
            while (last.Value == "-" || last.Value == "not" || last.Value == "#")
            {
                Read(input, ref debug);

                if (last.Value == "-")
                {
                    ex.Push(new UnaryInfo(1, last.StartPos, last.StartLine));
                }
                else if (last.Value == "not")
                {
                    ex.Push(new UnaryInfo(2, last.StartPos, last.StartLine));
                }
                else
                {
                    Contract.Assert(last.Value == "#");
                    ex.Push(new UnaryInfo(3, last.StartPos, last.StartLine));
                }
                last = input.Peek();
            }

            // check for literals
            last = input.Peek();
            int over = -1;
            if (last.Value != null)
            {
                NumberFormatInfo ni = CultureInfo.CurrentCulture.NumberFormat;
                if (last.Value != "..." && (char.IsNumber(last.Value, 0) || last.Value.StartsWith(ni.NumberDecimalSeparator, StringComparison.CurrentCulture)))
                {
                    Read(input, ref debug); // read the number.
                    try
                    {
                        o = new LiteralItem(double.Parse(last.Value, CultureInfo.CurrentCulture)) { Debug = last };
                    }
                    catch (FormatException e)
                    {
                        throw new SyntaxException(Resources.BadNumberFormat, input.Name, last, e);
                    }
                }
                else if (last.Value.StartsWith("&", StringComparison.Ordinal))
                {
                    Read(input, ref debug);
                    try
                    {
                        o = new LiteralItem(Convert.ToDouble(long.Parse(last.Value.Substring(1),
                            NumberStyles.AllowHexSpecifier, CultureInfo.CurrentCulture))) { Debug = last };
                    }
                    catch (FormatException e)
                    {
                        throw new SyntaxException(Resources.BadNumberFormat, input.Name, last, e);
                    }
                }
                else if (last.Value.StartsWith("\"", StringComparison.Ordinal))
                {
                    Read(input, ref debug);
                    o = new LiteralItem(last.Value.Substring(1)) { Debug = last };
                }
                else if (last.Value.StartsWith("{", StringComparison.Ordinal))
                {
                    o = ReadTable(input, ref debug);
                }
                else if (last.Value == "(")
                {
                    Read(input, ref debug);
                    o = ReadExp(input, ref debug);
                    last = Read(input, ref debug);
                    if (last.Value != ")")
                        throw new SyntaxException(string.Format(Resources.TokenInvalidExpecting, last.Value, "expression", ")"),
                            input.Name, last);
                }
                else if (last.Value == "true")
                {
                    Read(input, ref debug);
                    o = new LiteralItem(true) { Debug = last };
                }
                else if (last.Value == "false")
                {
                    Read(input, ref debug);
                    o = new LiteralItem(false) { Debug = last };
                }
                else if (last.Value == "nil")
                {
                    Read(input, ref debug);
                    o = new LiteralItem(null) { Debug = last };
                }
                else if (last.Value == "function")
//.........这里部分代码省略.........

        /// <summary>
        /// Reads a goto statement from the input.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="prev">The token to append what is read into.</param>
        /// <returns>The object that was read.</returns>
        protected virtual IParseStatement ReadGoto(ITokenizer input, ref Token prev)
        {
            var debug = input.Read(); // read 'goto'
            if (debug.Value != "goto")
                throw new InvalidOperationException(string.Format(Resources.MustBeOn, "goto", "ReadGoto"));
            
            // read the target
            var name = Read(input, ref debug);
            if (!IsName(name.Value))
                throw new SyntaxException(
                    string.Format(Resources.TokenNotAName, "goto", name.Value),
                    input.Name, debug);

            if (_reserved.Contains(name.Value))
                throw new SyntaxException(
                    string.Format(Resources.TokenReserved, name.Value),
                    input.Name, name);

            prev.Append(debug);
            return new GotoItem(name.Value) { Debug = debug };
        }

        /// <summary>
        /// Reads a do statement from the input.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="prev">The token to append what is read into.</param>
        /// <returns>The object that was read.</returns>
        protected virtual IParseStatement ReadDo(ITokenizer input, ref Token prev)
        {
            var debug = input.Read(); // read 'do'
            if (debug.Value != "do")
                throw new InvalidOperationException(string.Format(Resources.MustBeOn, "do", "ReadDo"));

            // read the block
            var ret = ReadBlock(input, ref debug);

            // ensure that it ends with 'end'
            Token end = Read(input, ref debug);
            if (end.Value != "end")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, end.Value, "do", "end"),
                    input.Name, end);
            
            prev.Append(debug);
            return ret;
        }

        /// <summary>
        /// Reads a label statement from the input.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="prev">The token to append what is read into.</param>
        /// <returns>The object that was read.</returns>
        protected virtual IParseStatement ReadLabel(ITokenizer input, ref Token prev)
        {
            var debug = input.Read(); // read '::'
            if (debug.Value != "::")
                throw new InvalidOperationException(string.Format(Resources.MustBeOn, "::", "ReadLabel"));

            // read the label
            Token label = Read(input, ref debug);

            // read '::'
            var name = Read(input, ref debug);
            if (name.Value != "::")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, name.Value, "label", "::"),
                    input.Name, debug);

            prev.Append(debug);
            return new LabelItem(label.Value) { Debug = debug };
        }

        /// <summary>
        /// Reads a break statement from the input.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="prev">The token to append what is read into.</param>
        /// <returns>The object that was read.</returns>
        protected virtual IParseStatement ReadBreak(ITokenizer input, ref Token prev)
        {
            var ret = input.Read();
            if (ret.Value != "break")
                throw new InvalidOperationException(string.Format(Resources.MustBeOn, "break", "ReadBreak"));

            prev.Append(ret);
            return new GotoItem("<break>") { Debug = ret };
        }

        /// <summary>
        /// Reads an if statement from the input.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="prev">The token to append what is read into.</param>
        /// <returns>The object that was read.</returns>
        protected virtual IParseStatement ReadIf(ITokenizer input, ref Token prev)
        {
            var debug = input.Read(); // read 'if'
            IfItem i = new IfItem();
            if (debug.Value != "if")
                throw new InvalidOperationException(string.Format(Resources.MustBeOn, "if", "ReadIf"));

            // read the initial expression
            i.Exp = ReadExp(input, ref debug);

            // read 'then'
            var name = Read(input, ref debug);
            if (name.Value != "then")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalid, name.Value, "if"),
                    input.Name, debug);

            // read the block
            var readBlock = ReadBlock(input, ref debug);
            i.Block = readBlock;

            // handle elseif(s)
            while ((name = input.Peek()).Value == "elseif")
            {
                Read(input, ref debug); // read 'elseif'

                // read the expression
                var readExp = ReadExp(input, ref debug);

                // read 'then'
                name = Read(input, ref debug);
                if (name.Value != "then")
                    throw new SyntaxException(
                        string.Format(Resources.TokenInvalid, name.Value, "elseif"),
                        input.Name, debug);

                // read the block
                readBlock = ReadBlock(input, ref debug);
                i.AddElse(readExp, readBlock);
            }

            // handle else
            if (name.Value != "else" && name.Value != "end")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalid, name.Value, "if"),
                    input.Name, debug);
            if (name.Value == "else")
            {
                Read(input, ref debug); // read 'else'

                // read the block
                readBlock = ReadBlock(input, ref debug);
                i.ElseBlock = readBlock;
            }

            // read 'end'
            name = Read(input, ref debug);
            if (name.Value != "end")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalid, name.Value, "if"),
                    input.Name, debug);

            prev.Append(debug);
            i.Debug = debug;
            return i;
        }

        /// <summary>
        /// Reads a repeat statement from the input.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="prev">The token to append what is read into.</param>
        /// <returns>The object that was read.</returns>
        protected virtual IParseStatement ReadRepeat(ITokenizer input, ref Token prev)
        {
            var debug = input.Read(); // read 'repeat'
            RepeatItem repeat = new RepeatItem();
            if (debug.Value != "repeat")
                throw new InvalidOperationException(string.Format(Resources.MustBeOn, "repeat", "ReadRepeat"));

            // read the block
            repeat.Block = ReadBlock(input, ref debug);

            // read 'until'
            var name = Read(input, ref debug);
            if (name.Value != "until")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, name.Value, "repeat", "until"),
                    input.Name, name);

            // read the expression
            repeat.Expression = ReadExp(input, ref debug);

            prev.Append(debug);
            repeat.Debug = debug;
            return repeat;
        }

        /// <summary>
        /// Reads a return statement from the input.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="prev">The token to append what is read into.</param>
        /// <returns>The object that was read.</returns>
        protected virtual ReturnItem ReadReturn(ITokenizer input, ref Token prev)
        {
            var debug = input.Read(); // read 'return'
            ReturnItem r = new ReturnItem();
            if (debug.Value != "return")
                throw new InvalidOperationException(string.Format(Resources.MustBeOn, "return", "ReadReturn"));

            var name = input.Peek();
            if (name.Value != "end" && name.Value != "until" && name.Value != "elseif" &&
                name.Value != "else")
            {
                r.AddExpression(ReadExp(input, ref debug));
                while (input.Peek().Value == ",")
                {
                    Read(input, ref debug); // read ','
                    r.AddExpression(ReadExp(input, ref debug));
                }

                if (input.Peek().Value == ";")
                {
                    Read(input, ref debug); // read ';'
                }

                // look at the next token for validation but keep it in the
                //  reader for the parrent.
                name = input.Peek();
                if (name.Value != "end" && name.Value != "until" && name.Value != "elseif" &&
                    name.Value != "else" && !IsNullOrWhiteSpace(name.Value))
                    throw new SyntaxException(
                        Resources.ReturnAtEnd,
                        input.Name, debug);
            }

            prev.Append(debug);
            r.Debug = debug;
            return r;
        }

        /// <summary>
        /// Reads a for statement from the input.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="prev">The token to append what is read into.</param>
        /// <returns>The object that was read.</returns>
        protected virtual IParseStatement ReadFor(ITokenizer input, ref Token prev)
        {
            var debug = input.Read(); // read 'for'
            if (debug.Value != "for")
                throw new InvalidOperationException(string.Format(Resources.MustBeOn, "for", "ReadFor"));

            // read a name
            var name = Read(input, ref debug);
            if (!IsName(name.Value))
                throw new SyntaxException(
                    string.Format(Resources.TokenNotAName, "for", name.Value),
                    input.Name, name);
            if (_reserved.Contains(name.Value))
                throw new SyntaxException(
                    string.Format(Resources.TokenReserved, name.Value),
                    input.Name, name);

            // numeric for
            if (input.Peek().Value == "=")
            {
                var ret = ReadNumberFor(input, ref debug, name);
                prev.Append(debug);
                return ret;
            }
            // generic for statement
            else
            {
                var ret = ReadGenericFor(input, ref debug, name);
                prev.Append(debug);
                return ret;
            }
        }

        /// <summary>
        /// Reads a class statement from the input.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="prev">The token to append what is read into.</param>
        /// <returns>The object that was read.</returns>
        protected virtual IParseStatement ReadClass(ITokenizer input, ref Token prev)
        {
            var debug = input.Read(); // read 'class'
            string sname = null;
            List<string> imp = new List<string>();
            if (debug.Value != "class")
                throw new InvalidOperationException(string.Format(Resources.MustBeOn, "class", "ReadClass"));

            if (input.Peek().Value.StartsWith("'", StringComparison.Ordinal) ||
                input.Peek().Value.StartsWith("\"", StringComparison.Ordinal))
            {
                var name = Read(input, ref debug);
                sname = name.Value.Substring(1);
                if (input.Peek().Value == "(")
                {
                    Read(input, ref debug); // read '('
                    while (input.Peek().Value != ")")
                    {
                        // read the name
                        name = Read(input, ref debug);
                        if (!IsName(name.Value))
                            throw new SyntaxException(
                                string.Format(Resources.TokenNotAName, "class", name.Value),
                                input.Name, name);
                        imp.Add(name.Value);

                        // read ','
                        name = Read(input, ref debug);
                        if (name.Value != ",")
                            throw new SyntaxException(
                                string.Format(Resources.TokenInvalid, name.Value, "class"),
                                input.Name, name);
                    }
                    Read(input, ref debug); // read ')'
                }
            }
            else
            {
                var name = Read(input, ref debug);
                sname = name.Value;
                if (!IsName(sname))
                    throw new SyntaxException(
                        string.Format(Resources.TokenNotAName, "class", name.Value),
                        input.Name, name);
                if (input.Peek().Value == ":")
                {
                    do
                    {
                        // simply include the '.' in the name.
                        string n = "";
                        do
                        {
                            Read(input, ref debug); // read ':' or ','
                            n += (n == "" ? "" : ".") + Read(input, ref debug).Value;
                        } while (input.Peek().Value == ".");

                        imp.Add(n);
                    } while (input.Peek().Value == ",");
                }
            }

            prev.Append(debug);
            return new ClassDefItem(sname, imp.ToArray()) { Debug = debug };
        }

        /// <summary>
        /// Reads a local statement from the input.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="prev">The token to append what is read into.</param>
        /// <returns>The object that was read.</returns>
        protected virtual IParseStatement ReadLocal(ITokenizer input, ref Token prev)
        {
            var debug = input.Read(); // read 'local'
            if (debug.Value != "local")
                throw new InvalidOperationException(string.Format(Resources.MustBeOn, "local", "ReadLocal"));

            var name = input.Peek();
            if (name.Value == "function")
            {
                prev.Append(debug);
                return ReadFunctionHelper(input, ref prev, true, true);
            }
            else
            {
                Read(input, ref debug); // read name
                if (!IsName(name.Value))
                    throw new SyntaxException(
                        string.Format(Resources.TokenNotAName, "local", name.Value),
                        input.Name, name);
                if (_reserved.Contains(name.Value))
                    throw new SyntaxException(
                        string.Format(Resources.TokenReserved, name.Value),
                        input.Name, name);

                var i = ReadAssignment(input, ref debug, true, new NameItem(name.Value) { Debug = name });
                prev.Append(debug);
                return i;
            }
        }

 /// <summary>
 /// Reads a token from the tokenizer and appends the read
 /// value to the given token.
 /// </summary>
 /// <param name="input">Where to get the input from.</param>
 /// <param name="token">A token to append the read token to.</param>
 /// <returns>The token that was read.</returns>
 protected static Token Read(ITokenizer input, ref Token token)
 {
     Token ret = input.Read();
     token.Append(ret);
     return ret;
 }

        /// <summary>
        /// Reads a table from the input.  Input must be either on the starting '{'.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="token">The token to append the read Tokenm to.</param>
        /// <returns>The table that was read.</returns>
        protected virtual TableItem ReadTable(ITokenizer input, ref Token token)
        {
            Token debug = input.Read();
            if (debug.Value != "{")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, debug.Value, "table", "{"),
                    input.Name, debug);

            TableItem ret = new TableItem();
            Token last = input.Peek();
            while (last.Value != "}")
            {
                if (last.Value == "[")
                {
                    Read(input, ref debug); // read the "["

                    var temp = ReadExp(input, ref debug);
                    if (temp == null)
                        throw new SyntaxException(string.Format(Resources.InvalidDefinition, "table"),
                            input.Name, debug);

                    // read ']'
                    last = Read(input, ref debug);
                    if (last.Value != "]")
                        throw new SyntaxException(
                            string.Format(Resources.TokenInvalidExpecting, last.Value, "table", "]"), 
                            input.Name, last);

                    // read '='
                    last = Read(input, ref debug);
                    if (last.Value != "=")
                        throw new SyntaxException(
                            string.Format(Resources.TokenInvalidExpecting, last.Value, "table", "="), 
                            input.Name, last);

                    // read the expression
                    var val = ReadExp(input, ref debug);
                    if (val == null)
                        throw new SyntaxException(string.Format(Resources.InvalidDefinition, "table"),
                            input.Name, debug);

                    ret.AddItem(temp, val);
                }
                else
                {
                    var val = ReadExp(input, ref debug);
                    if (input.Peek().Value == "=")
                    {
                        Read(input, ref debug); // read '='

                        NameItem name = val as NameItem;
                        if (name == null)
                            throw new SyntaxException(string.Format(Resources.InvalidDefinition, "table"),
                                input.Name, debug);

                        // read the expression
                        var exp = ReadExp(input, ref debug);
                        ret.AddItem(new LiteralItem(name.Name), exp);
                    }
                    else
                    {
                        ret.AddItem(null, val);
                    }
                }

                if (input.Peek().Value != "," && input.Peek().Value != ";")
                    break;
                else
                    Read(input, ref debug);
                last = input.Peek();
            } // end While

            Token end = Read(input, ref debug); // read the "}"
            if (end.Value != "}")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, end.Value, "table", "}"), 
                    input.Name, end);

            ret.Debug = debug;
            token.Append(debug);
            return ret;
        }

        /// <summary>
        /// Reads a function from the input.  Input must either be on the word 'function' or
        /// on the next token.  If it is on 'function' and canName is true, it will give
        /// the function the read name; otherwise it will give it a null name.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="token">The token to append the read Token to.</param>
        /// <param name="canName">True if the function can have a name, otherwise false.</param>
        /// <param name="local">True if this function is a local definition, otherwise false.</param>
        /// <returns>The function definition that was read.</returns>
        protected virtual FuncDefItem ReadFunctionHelper(ITokenizer input, ref Token token, bool canName, bool local)
        {
            IParseVariable name = null;
            string inst = null;
            Token last = input.Peek(), debug = last;
            if (last.Value == "function")
            {
                input.Read(); // read 'function'
                last = input.Peek();
                if (IsName(last.Value))
                {
                    Token nameTok = input.Read(); // read name
                    name = new NameItem(last.Value) { Debug = last };

                    // handle indexers
                    last = input.Peek();
                    while (last.Value == ".")
                    {
                        Read(input, ref nameTok); // read '.'
                        last = input.Peek();
                        if (!IsName(last.Value))
                            break;

                        name = new IndexerItem(name, new LiteralItem(last.Value) { Debug = last }) { Debug = nameTok };
                        Read(input, ref nameTok);
                    }

                    if (input.Peek().Value == ":")
                    {
                        Read(input, ref nameTok);
                        inst = Read(input, ref nameTok).Value;
                        if (!IsName(inst))
                            throw new SyntaxException(string.Format(Resources.TokenInvalid, last.Value, "function"),
                                input.Name, last);
                    }
                    debug.Append(nameTok);
                }
            }
            if (name != null && !canName)
                throw new SyntaxException(Resources.FunctionCantHaveName, input.Name, debug);

            FuncDefItem ret = new FuncDefItem(name, local);
            ret.InstanceName = inst;
            last = Read(input, ref debug);
            if (last.Value != "(")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, last.Value, "function", "("),
                    input.Name, last);

            last = input.Peek();
            while (last.Value != ")")
            {
                Token temp = Read(input, ref debug); // read the name
                if (!IsName(last.Value) && last.Value != "...")
                    throw new SyntaxException(string.Format(Resources.TokenInvalid, last.Value, "function"),
                        input.Name, temp);
                ret.AddArgument(new NameItem(last.Value) { Debug = last });

                last = input.Peek();
                if (last.Value == ",")
                    Read(input, ref debug);
                else if (last.Value != ")")
                    throw new SyntaxException(
                        string.Format(Resources.TokenInvalidExpecting2, last.Value, "function", ",", ")"),
                        input.Name, last);

                last = input.Peek();
            }
            if (last.Value != ")")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, last.Value, "function", ")"),
                    input.Name, last);
            Read(input, ref debug); // read ')'

            BlockItem chunk = ReadBlock(input, ref debug);
            chunk.Return = chunk.Return ?? new ReturnItem();
            ret.Block = chunk;
            last = Read(input, ref debug);
            if (last.Value != "end")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, last.Value, "function", "end"),
                    input.Name, last);

            token.Append(debug);
            ret.Debug = debug;
            return ret;
        }