C# ITokenizer.Peek方法代码示例

        /// <summary>
        /// Read parse trees from a Reader.
        /// </summary>
        /// <param name="input">The Reader</param>
        /// <param name="tf">TreeFactory -- factory to create some kind of Tree</param>
        /// <param name="tn">the method of normalizing trees</param>
        /// <param name="st">Tokenizer that divides up Reader</param>
        public PennTreeReader(TextReader input, ITreeFactory tf, TreeNormalizer tn, ITokenizer<string> st)
        {
            reader = input;
            treeFactory = tf;
            treeNormalizer = tn;
            tokenizer = st;

            // check for whacked out headers still present in Brown corpus in Treebank 3
            string first = (st.HasNext() ? st.Peek() : null);
            if (first != null && first.StartsWith("*x*x*x"))
            {
                int foundCount = 0;
                while (foundCount < 4 && st.HasNext())
                {
                    first = st.Next();
                    if (first != null && first.StartsWith("*x*x*x"))
                    {
                        foundCount++;
                    }
                }
            }
        }

        /// <summary>
        /// Reads part of a numerical for loop from the input.  The input is 
        /// currently on the equals sign '=' and the debug token currently
        /// contains the parts read for the 'for' loop.  'name' contains the
        /// name of the variable.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="debug">The token that currently holds what was read
        /// so far in the for statement and should after contain the entire loop.</param>
        /// <param name="name">The token that contains the name of the variable.</param>
        /// <returns>The loop object that was read.</returns>
        protected virtual ForNumItem ReadNumberFor(ITokenizer input, ref Token debug, Token name)
        {
            // read "="
            var temp = Read(input, ref debug);
            if (temp.Value != "=")
                throw new InvalidOperationException(string.Format(Resources.MustBeOn, "=", "ReadNumberFor"));

            // get the 'start' value
            var start = ReadExp(input, ref debug);

            // read ','
            temp = Read(input, ref debug);
            if (temp.Value != ",")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, temp.Value, "for", ","),
                    input.Name, temp);

            // get the 'limit'
            var limit = ReadExp(input, ref debug);

            // read ','
            IParseExp step = null;
            if (input.Peek().Value == ",")
            {
                Read(input, ref debug);

                // read the 'step'
                step = ReadExp(input, ref debug);
            }

            ForNumItem i = new ForNumItem(new NameItem(name.Value) { Debug = name }, start, limit, step);

            // check for 'do'
            name = Read(input, ref debug);
            if (name.Value != "do")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, name.Value, "for", "do"),
                    input.Name, name);

            // read the block
            i.Block = ReadBlock(input, ref debug);

            // read 'end'
            name = Read(input, ref debug);
            if (name.Value != "end")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, name.Value, "for", "end"),
                    input.Name, name);

            i.Debug = debug;
            return i;
        }

        /// <summary>
        /// Reads a prefix-expression from the input.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="token">The token to append the total token onto.</param>
        /// <returns>The expression that was read.</returns>
        protected virtual IParseExp ReadPrefixExp(ITokenizer input, ref Token token)
        {
            Stack<UnaryInfo> ex = new Stack<UnaryInfo>();
            IParseExp o = null;
            Token last, debug = input.Peek();
            debug.Value = "";

            // check for unary operators
            last = input.Peek();
            while (last.Value == "-" || last.Value == "not" || last.Value == "#")
            {
                Read(input, ref debug);

                if (last.Value == "-")
                {
                    ex.Push(new UnaryInfo(1, last.StartPos, last.StartLine));
                }
                else if (last.Value == "not")
                {
                    ex.Push(new UnaryInfo(2, last.StartPos, last.StartLine));
                }
                else
                {
                    Contract.Assert(last.Value == "#");
                    ex.Push(new UnaryInfo(3, last.StartPos, last.StartLine));
                }
                last = input.Peek();
            }

            // check for literals
            last = input.Peek();
            int over = -1;
            if (last.Value != null)
            {
                NumberFormatInfo ni = CultureInfo.CurrentCulture.NumberFormat;
                if (last.Value != "..." && (char.IsNumber(last.Value, 0) || last.Value.StartsWith(ni.NumberDecimalSeparator, StringComparison.CurrentCulture)))
                {
                    Read(input, ref debug); // read the number.
                    try
                    {
                        o = new LiteralItem(double.Parse(last.Value, CultureInfo.CurrentCulture)) { Debug = last };
                    }
                    catch (FormatException e)
                    {
                        throw new SyntaxException(Resources.BadNumberFormat, input.Name, last, e);
                    }
                }
                else if (last.Value.StartsWith("&", StringComparison.Ordinal))
                {
                    Read(input, ref debug);
                    try
                    {
                        o = new LiteralItem(Convert.ToDouble(long.Parse(last.Value.Substring(1),
                            NumberStyles.AllowHexSpecifier, CultureInfo.CurrentCulture))) { Debug = last };
                    }
                    catch (FormatException e)
                    {
                        throw new SyntaxException(Resources.BadNumberFormat, input.Name, last, e);
                    }
                }
                else if (last.Value.StartsWith("\"", StringComparison.Ordinal))
                {
                    Read(input, ref debug);
                    o = new LiteralItem(last.Value.Substring(1)) { Debug = last };
                }
                else if (last.Value.StartsWith("{", StringComparison.Ordinal))
                {
                    o = ReadTable(input, ref debug);
                }
                else if (last.Value == "(")
                {
                    Read(input, ref debug);
                    o = ReadExp(input, ref debug);
                    last = Read(input, ref debug);
                    if (last.Value != ")")
                        throw new SyntaxException(string.Format(Resources.TokenInvalidExpecting, last.Value, "expression", ")"),
                            input.Name, last);
                }
                else if (last.Value == "true")
                {
                    Read(input, ref debug);
                    o = new LiteralItem(true) { Debug = last };
                }
                else if (last.Value == "false")
                {
                    Read(input, ref debug);
                    o = new LiteralItem(false) { Debug = last };
                }
                else if (last.Value == "nil")
                {
                    Read(input, ref debug);
                    o = new LiteralItem(null) { Debug = last };
                }
                else if (last.Value == "function")
//.........这里部分代码省略.........

        /// <summary>
        /// Reads an assignment statement from the input.  The input is currently
        /// after the first name, on the comma or equal sign.  The debug token
        /// contains the name and should contain the entire statement.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="debug">Currently contains the first name, and after
        /// should contain the entire statement.</param>
        /// <param name="local">True if this is a local definition, otherwise false.</param>
        /// <param name="variable">The first variable that was read.</param>
        /// <returns>The statement that was read.</returns>
        protected virtual AssignmentItem ReadAssignment(ITokenizer input, ref Token debug, bool local, IParseVariable variable)
        {
            // read each of the variable names
            AssignmentItem assign = new AssignmentItem(local);
            assign.AddName(variable);
            while (input.Peek().Value == ",")
            {
                Read(input, ref debug); // read ','

                // read the left-hand-expression
                var exp = ReadExp(input, ref debug);
                if ((local && !(exp is NameItem)) || (!local && !(exp is IParseVariable)))
                    throw new SyntaxException(Resources.NameOrExpForVar, input.Name, debug);
                assign.AddName((IParseVariable)exp);
            }

            // read the initial values
            if (input.Peek().Value == "=")
            {
                Read(input, ref debug); // read '='
                assign.AddItem(ReadExp(input, ref debug));

                while (input.Peek().Value == ",")
                {
                    Read(input, ref debug); // read ','
                    assign.AddItem(ReadExp(input, ref debug));
                }
            }
            else if (!local)
                throw new SyntaxException(
                    string.Format(Resources.InvalidDefinition, "assignment"),
                    input.Name, debug);

            assign.Debug = debug;
            return assign;
        }

        /// <summary>
        /// Reads part of a generic for loop from the input.  The input is
        /// currently on the token after the first name and debug contains
        /// the parts read for the 'for' loop.  'name' contains the name of
        /// the first variable.
        /// </summary>
        /// <param name="input">Where to read input from/</param>
        /// <param name="debug">The token that currently holds what was read
        /// so far in the for statement and should after contain the entire loop.</param>
        /// <param name="name">The token that contains the name of the variable.</param>
        /// <returns>The loop object that was read.</returns>
        protected virtual ForGenItem ReadGenericFor(ITokenizer input, ref Token debug, Token name)
        {
            // read the variables
            List<NameItem> names = new List<NameItem>();
            names.Add(new NameItem(name.Value) { Debug = name });

            while (input.Peek().Value == ",")
            {
                Read(input, ref debug); // read ','

                // read the name
                name = Read(input, ref debug);
                if (!IsName(name.Value))
                    throw new SyntaxException(
                        string.Format(Resources.TokenNotAName, "for", name.Value),
                        input.Name, name);
                if (_reserved.Contains(name.Value))
                    throw new SyntaxException(
                        string.Format(Resources.TokenReserved, name.Value),
                        input.Name, name);

                names.Add(new NameItem(name.Value) { Debug = name });
            }

            // check for 'in'
            name = Read(input, ref debug);
            if (name.Value != "in")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, name.Value, "for", "in"),
                    input.Name, name);

            // read the expression-list
            ForGenItem f = new ForGenItem(names);
            f.AddExpression(ReadExp(input, ref debug));
            while (input.Peek().Value == ",")
            {
                Read(input, ref debug); // read ","
                f.AddExpression(ReadExp(input, ref debug));
            }

            // check for 'do'
            name = Read(input, ref debug);
            if (name.Value != "do")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, name.Value, "for", "do"),
                    input.Name, name);

            // read the chunk
            f.Block = ReadBlock(input, ref debug);

            // read 'end'
            name = Read(input, ref debug);
            if (name.Value != "end")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, name.Value, "for", "end"),
                    input.Name, name);

            f.Debug = debug;
            return f;
        }

        /// <summary>
        /// Reads a for statement from the input.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="prev">The token to append what is read into.</param>
        /// <returns>The object that was read.</returns>
        protected virtual IParseStatement ReadFor(ITokenizer input, ref Token prev)
        {
            var debug = input.Read(); // read 'for'
            if (debug.Value != "for")
                throw new InvalidOperationException(string.Format(Resources.MustBeOn, "for", "ReadFor"));

            // read a name
            var name = Read(input, ref debug);
            if (!IsName(name.Value))
                throw new SyntaxException(
                    string.Format(Resources.TokenNotAName, "for", name.Value),
                    input.Name, name);
            if (_reserved.Contains(name.Value))
                throw new SyntaxException(
                    string.Format(Resources.TokenReserved, name.Value),
                    input.Name, name);

            // numeric for
            if (input.Peek().Value == "=")
            {
                var ret = ReadNumberFor(input, ref debug, name);
                prev.Append(debug);
                return ret;
            }
            // generic for statement
            else
            {
                var ret = ReadGenericFor(input, ref debug, name);
                prev.Append(debug);
                return ret;
            }
        }

        /// <summary>
        /// Reads an if statement from the input.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="prev">The token to append what is read into.</param>
        /// <returns>The object that was read.</returns>
        protected virtual IParseStatement ReadIf(ITokenizer input, ref Token prev)
        {
            var debug = input.Read(); // read 'if'
            IfItem i = new IfItem();
            if (debug.Value != "if")
                throw new InvalidOperationException(string.Format(Resources.MustBeOn, "if", "ReadIf"));

            // read the initial expression
            i.Exp = ReadExp(input, ref debug);

            // read 'then'
            var name = Read(input, ref debug);
            if (name.Value != "then")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalid, name.Value, "if"),
                    input.Name, debug);

            // read the block
            var readBlock = ReadBlock(input, ref debug);
            i.Block = readBlock;

            // handle elseif(s)
            while ((name = input.Peek()).Value == "elseif")
            {
                Read(input, ref debug); // read 'elseif'

                // read the expression
                var readExp = ReadExp(input, ref debug);

                // read 'then'
                name = Read(input, ref debug);
                if (name.Value != "then")
                    throw new SyntaxException(
                        string.Format(Resources.TokenInvalid, name.Value, "elseif"),
                        input.Name, debug);

                // read the block
                readBlock = ReadBlock(input, ref debug);
                i.AddElse(readExp, readBlock);
            }

            // handle else
            if (name.Value != "else" && name.Value != "end")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalid, name.Value, "if"),
                    input.Name, debug);
            if (name.Value == "else")
            {
                Read(input, ref debug); // read 'else'

                // read the block
                readBlock = ReadBlock(input, ref debug);
                i.ElseBlock = readBlock;
            }

            // read 'end'
            name = Read(input, ref debug);
            if (name.Value != "end")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalid, name.Value, "if"),
                    input.Name, debug);

            prev.Append(debug);
            i.Debug = debug;
            return i;
        }

        /// <summary>
        /// Reads a class statement from the input.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="prev">The token to append what is read into.</param>
        /// <returns>The object that was read.</returns>
        protected virtual IParseStatement ReadClass(ITokenizer input, ref Token prev)
        {
            var debug = input.Read(); // read 'class'
            string sname = null;
            List<string> imp = new List<string>();
            if (debug.Value != "class")
                throw new InvalidOperationException(string.Format(Resources.MustBeOn, "class", "ReadClass"));

            if (input.Peek().Value.StartsWith("'", StringComparison.Ordinal) ||
                input.Peek().Value.StartsWith("\"", StringComparison.Ordinal))
            {
                var name = Read(input, ref debug);
                sname = name.Value.Substring(1);
                if (input.Peek().Value == "(")
                {
                    Read(input, ref debug); // read '('
                    while (input.Peek().Value != ")")
                    {
                        // read the name
                        name = Read(input, ref debug);
                        if (!IsName(name.Value))
                            throw new SyntaxException(
                                string.Format(Resources.TokenNotAName, "class", name.Value),
                                input.Name, name);
                        imp.Add(name.Value);

                        // read ','
                        name = Read(input, ref debug);
                        if (name.Value != ",")
                            throw new SyntaxException(
                                string.Format(Resources.TokenInvalid, name.Value, "class"),
                                input.Name, name);
                    }
                    Read(input, ref debug); // read ')'
                }
            }
            else
            {
                var name = Read(input, ref debug);
                sname = name.Value;
                if (!IsName(sname))
                    throw new SyntaxException(
                        string.Format(Resources.TokenNotAName, "class", name.Value),
                        input.Name, name);
                if (input.Peek().Value == ":")
                {
                    do
                    {
                        // simply include the '.' in the name.
                        string n = "";
                        do
                        {
                            Read(input, ref debug); // read ':' or ','
                            n += (n == "" ? "" : ".") + Read(input, ref debug).Value;
                        } while (input.Peek().Value == ".");

                        imp.Add(n);
                    } while (input.Peek().Value == ",");
                }
            }

            prev.Append(debug);
            return new ClassDefItem(sname, imp.ToArray()) { Debug = debug };
        }

        /// <summary>
        /// Reads a return statement from the input.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="prev">The token to append what is read into.</param>
        /// <returns>The object that was read.</returns>
        protected virtual ReturnItem ReadReturn(ITokenizer input, ref Token prev)
        {
            var debug = input.Read(); // read 'return'
            ReturnItem r = new ReturnItem();
            if (debug.Value != "return")
                throw new InvalidOperationException(string.Format(Resources.MustBeOn, "return", "ReadReturn"));

            var name = input.Peek();
            if (name.Value != "end" && name.Value != "until" && name.Value != "elseif" &&
                name.Value != "else")
            {
                r.AddExpression(ReadExp(input, ref debug));
                while (input.Peek().Value == ",")
                {
                    Read(input, ref debug); // read ','
                    r.AddExpression(ReadExp(input, ref debug));
                }

                if (input.Peek().Value == ";")
                {
                    Read(input, ref debug); // read ';'
                }

                // look at the next token for validation but keep it in the
                //  reader for the parrent.
                name = input.Peek();
                if (name.Value != "end" && name.Value != "until" && name.Value != "elseif" &&
                    name.Value != "else" && !IsNullOrWhiteSpace(name.Value))
                    throw new SyntaxException(
                        Resources.ReturnAtEnd,
                        input.Name, debug);
            }

            prev.Append(debug);
            r.Debug = debug;
            return r;
        }

        /// <summary>
        /// Reads a local statement from the input.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="prev">The token to append what is read into.</param>
        /// <returns>The object that was read.</returns>
        protected virtual IParseStatement ReadLocal(ITokenizer input, ref Token prev)
        {
            var debug = input.Read(); // read 'local'
            if (debug.Value != "local")
                throw new InvalidOperationException(string.Format(Resources.MustBeOn, "local", "ReadLocal"));

            var name = input.Peek();
            if (name.Value == "function")
            {
                prev.Append(debug);
                return ReadFunctionHelper(input, ref prev, true, true);
            }
            else
            {
                Read(input, ref debug); // read name
                if (!IsName(name.Value))
                    throw new SyntaxException(
                        string.Format(Resources.TokenNotAName, "local", name.Value),
                        input.Name, name);
                if (_reserved.Contains(name.Value))
                    throw new SyntaxException(
                        string.Format(Resources.TokenReserved, name.Value),
                        input.Name, name);

                var i = ReadAssignment(input, ref debug, true, new NameItem(name.Value) { Debug = name });
                prev.Append(debug);
                return i;
            }
        }

        /// <summary>
        /// Reads a block of code from the input.  Any end tokens
        /// should not be read and are handled by the parrent call
        /// (e.g. 'end' or 'until').
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="prev">The token to append the total token onto.</param>
        /// <returns>The item that was read.</returns>
        protected virtual BlockItem ReadBlock(ITokenizer input, ref Token prev)
        {
            BlockItem ret = new BlockItem();
            Token total = input.Peek();
            total.Value = "";
            Token name;

            while ((name = input.Peek()).Value != null)
            {
                if (Functions.ContainsKey(name.Value))
                {
                    var temp = Functions[name.Value](input, ref total);
                    ret.AddItem(temp);
                }
                else if (name.Value == "return")
                {
                    ret.Return = ReadReturn(input, ref total);
                    return ret;
                }
                else if (name.Value == ";")
                {
                    Read(input, ref total); // read ';'
                }
                else if (name.Value == "end" || name.Value == "else" || name.Value == "elseif" ||
                    name.Value == "until")
                {
                    // don'type read as it will be handled by the parrent
                    prev.Append(total); // don't add 'end' to the prev 
                    ret.Debug = total;  //   or the current block, this 
                    return ret;         //   end belongs to the parrent.
                }
                else
                {
                    Token debug = name;
                    debug.Value = "";
                    var exp = ReadExp(input, ref debug);
                    if (exp is FuncCallItem)
                    {
                        (exp as FuncCallItem).Statement = true;
                        ret.AddItem((FuncCallItem)exp);
                    }
                    else if (exp is LiteralItem)
                    {
                        throw new SyntaxException(
                            "A literal is not a variable.",
                            input.Name, debug);
                    }
                    else if (exp is NameItem || exp is IndexerItem)
                    {
                        var i = ReadAssignment(input, ref debug, false, (IParseVariable)exp);
                        ret.AddItem(i);
                    }
                    else
                        throw new SyntaxException(
                            string.Format(Resources.TokenStatement, name.Value),
                            input.Name, debug);

                    total.Append(debug);
                }
            } // end While

            // only gets here if this is the global function
            ret.Debug = total;
            ret.Return = ret.Return ?? new ReturnItem();
            return ret;
        }

        /// <summary>
        /// Reads a table from the input.  Input must be either on the starting '{'.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="token">The token to append the read Tokenm to.</param>
        /// <returns>The table that was read.</returns>
        protected virtual TableItem ReadTable(ITokenizer input, ref Token token)
        {
            Token debug = input.Read();
            if (debug.Value != "{")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, debug.Value, "table", "{"),
                    input.Name, debug);

            TableItem ret = new TableItem();
            Token last = input.Peek();
            while (last.Value != "}")
            {
                if (last.Value == "[")
                {
                    Read(input, ref debug); // read the "["

                    var temp = ReadExp(input, ref debug);
                    if (temp == null)
                        throw new SyntaxException(string.Format(Resources.InvalidDefinition, "table"),
                            input.Name, debug);

                    // read ']'
                    last = Read(input, ref debug);
                    if (last.Value != "]")
                        throw new SyntaxException(
                            string.Format(Resources.TokenInvalidExpecting, last.Value, "table", "]"), 
                            input.Name, last);

                    // read '='
                    last = Read(input, ref debug);
                    if (last.Value != "=")
                        throw new SyntaxException(
                            string.Format(Resources.TokenInvalidExpecting, last.Value, "table", "="), 
                            input.Name, last);

                    // read the expression
                    var val = ReadExp(input, ref debug);
                    if (val == null)
                        throw new SyntaxException(string.Format(Resources.InvalidDefinition, "table"),
                            input.Name, debug);

                    ret.AddItem(temp, val);
                }
                else
                {
                    var val = ReadExp(input, ref debug);
                    if (input.Peek().Value == "=")
                    {
                        Read(input, ref debug); // read '='

                        NameItem name = val as NameItem;
                        if (name == null)
                            throw new SyntaxException(string.Format(Resources.InvalidDefinition, "table"),
                                input.Name, debug);

                        // read the expression
                        var exp = ReadExp(input, ref debug);
                        ret.AddItem(new LiteralItem(name.Name), exp);
                    }
                    else
                    {
                        ret.AddItem(null, val);
                    }
                }

                if (input.Peek().Value != "," && input.Peek().Value != ";")
                    break;
                else
                    Read(input, ref debug);
                last = input.Peek();
            } // end While

            Token end = Read(input, ref debug); // read the "}"
            if (end.Value != "}")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, end.Value, "table", "}"), 
                    input.Name, end);

            ret.Debug = debug;
            token.Append(debug);
            return ret;
        }

        /// <summary>
        /// Reads a function from the input.  Input must either be on the word 'function' or
        /// on the next token.  If it is on 'function' and canName is true, it will give
        /// the function the read name; otherwise it will give it a null name.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="token">The token to append the read Token to.</param>
        /// <param name="canName">True if the function can have a name, otherwise false.</param>
        /// <param name="local">True if this function is a local definition, otherwise false.</param>
        /// <returns>The function definition that was read.</returns>
        protected virtual FuncDefItem ReadFunctionHelper(ITokenizer input, ref Token token, bool canName, bool local)
        {
            IParseVariable name = null;
            string inst = null;
            Token last = input.Peek(), debug = last;
            if (last.Value == "function")
            {
                input.Read(); // read 'function'
                last = input.Peek();
                if (IsName(last.Value))
                {
                    Token nameTok = input.Read(); // read name
                    name = new NameItem(last.Value) { Debug = last };

                    // handle indexers
                    last = input.Peek();
                    while (last.Value == ".")
                    {
                        Read(input, ref nameTok); // read '.'
                        last = input.Peek();
                        if (!IsName(last.Value))
                            break;

                        name = new IndexerItem(name, new LiteralItem(last.Value) { Debug = last }) { Debug = nameTok };
                        Read(input, ref nameTok);
                    }

                    if (input.Peek().Value == ":")
                    {
                        Read(input, ref nameTok);
                        inst = Read(input, ref nameTok).Value;
                        if (!IsName(inst))
                            throw new SyntaxException(string.Format(Resources.TokenInvalid, last.Value, "function"),
                                input.Name, last);
                    }
                    debug.Append(nameTok);
                }
            }
            if (name != null && !canName)
                throw new SyntaxException(Resources.FunctionCantHaveName, input.Name, debug);

            FuncDefItem ret = new FuncDefItem(name, local);
            ret.InstanceName = inst;
            last = Read(input, ref debug);
            if (last.Value != "(")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, last.Value, "function", "("),
                    input.Name, last);

            last = input.Peek();
            while (last.Value != ")")
            {
                Token temp = Read(input, ref debug); // read the name
                if (!IsName(last.Value) && last.Value != "...")
                    throw new SyntaxException(string.Format(Resources.TokenInvalid, last.Value, "function"),
                        input.Name, temp);
                ret.AddArgument(new NameItem(last.Value) { Debug = last });

                last = input.Peek();
                if (last.Value == ",")
                    Read(input, ref debug);
                else if (last.Value != ")")
                    throw new SyntaxException(
                        string.Format(Resources.TokenInvalidExpecting2, last.Value, "function", ",", ")"),
                        input.Name, last);

                last = input.Peek();
            }
            if (last.Value != ")")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, last.Value, "function", ")"),
                    input.Name, last);
            Read(input, ref debug); // read ')'

            BlockItem chunk = ReadBlock(input, ref debug);
            chunk.Return = chunk.Return ?? new ReturnItem();
            ret.Block = chunk;
            last = Read(input, ref debug);
            if (last.Value != "end")
                throw new SyntaxException(
                    string.Format(Resources.TokenInvalidExpecting, last.Value, "function", "end"),
                    input.Name, last);

            token.Append(debug);
            ret.Debug = debug;
            return ret;
        }

        /// <summary>
        /// Reads an expression from the input and returns the
        /// item that was read.
        /// </summary>
        /// <param name="input">Where to read input from.</param>
        /// <param name="precedence">The precedence of the previous expression
        /// or -1 if a root.</param>
        /// <param name="token">The Token that represents the entire expression
        /// should be appended to this variable.</param>
        /// <returns>The expression that was read.</returns>
        protected virtual IParseExp ReadExp(ITokenizer input, ref Token token, int precedence = -1)
        {
            Token debug = input.Peek();
            debug.Value = "";
            IParseExp cur = ReadPrefixExp(input, ref debug);
            BinOpItem ret = null;

        start:
            Token last = input.Peek();
            BinaryOperationType type = GetOperationType(last.Value);
            int nPrec = GetPrecedence(type);
            if (nPrec != -1 && (precedence == -1 || precedence > nPrec))
            {
                Read(input, ref debug); // read the exp
                var temp = ReadExp(input, ref debug, nPrec);
                ret = new BinOpItem(ret ?? cur, type, temp);
                ret.Debug = debug;
                goto start;
            }

            token.Append(debug);
            return ret ?? cur;
        }