当前位置: 首页>>代码示例>>C++>>正文


C++ Tokenizer::eof方法代码示例

本文整理汇总了C++中Tokenizer::eof方法的典型用法代码示例。如果您正苦于以下问题:C++ Tokenizer::eof方法的具体用法?C++ Tokenizer::eof怎么用?C++ Tokenizer::eof使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在Tokenizer的用法示例。


在下文中一共展示了Tokenizer::eof方法的3个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的C++代码示例。

示例1: doEmit

        Token MapTokenEmitter::doEmit(Tokenizer& tokenizer) {
            while (!tokenizer.eof()) {
                size_t line = tokenizer.line();
                size_t column = tokenizer.column();
                const char* c = tokenizer.nextChar();
                switch (*c) {
                    case '/':
                        if (tokenizer.peekChar() == '/') {
                            tokenizer.nextChar();
                            if (tokenizer.peekChar() == '/') {
                                tokenizer.nextChar(); // it's a TB comment
                            } else {
                                // eat everything up to and including the next newline
                                while (*tokenizer.nextChar() != '\n');
                            }
                        }
                        break;
                    case '{':
                        return Token(TokenType::OBrace, c, c + 1, tokenizer.offset(c), line, column);
                    case '}':
                        return Token(TokenType::CBrace, c, c + 1, tokenizer.offset(c), line, column);
                    case '(':
                        return Token(TokenType::OParenthesis, c, c + 1, tokenizer.offset(c), line, column);
                    case ')':
                        return Token(TokenType::CParenthesis, c, c + 1, tokenizer.offset(c), line, column);
                    case '[':
                        return Token(TokenType::OBracket, c, c + 1, tokenizer.offset(c), line, column);
                    case ']':
                        return Token(TokenType::CBracket, c, c + 1, tokenizer.offset(c), line, column);
                    case '"': { // quoted string
                        const char* begin = c;
                        const char* end;
                        tokenizer.quotedString(begin, end);
                        return Token(TokenType::String, begin, end, tokenizer.offset(begin), line, column);
                    }
                    default: { // whitespace, integer, decimal or word
                        if (isWhitespace(*c))
                            break;
                        
                        const char* begin = c;

                        // try to read a number
                        if (*c == '-' || isDigit(*c)) {
                            while (isDigit(*(c = tokenizer.nextChar())));
                            if (isDelimiter(*c)) {
                                if (!tokenizer.eof())
                                    tokenizer.pushChar();
                                return Token(TokenType::Integer, begin, c, tokenizer.offset(begin), line, column);
                            }
                        }
                        
                        // try to read a decimal (may start with '.')
                        if (*c == '.') {
                            while (isDigit(*(c = tokenizer.nextChar())));
                            if (isDelimiter(*c)) {
                                if (!tokenizer.eof())
                                    tokenizer.pushChar();
                                return Token(TokenType::Decimal, begin, c, tokenizer.offset(begin), line, column);
                            }
                        }
                        
                        // try to read decimal in scientific notation
                        if (*c == 'e') {
                            c = tokenizer.nextChar();
                            if (isDigit(*c) || *c == '+' || *c == '-') {
                                while (isDigit(*(c = tokenizer.nextChar())));
                                if (isDelimiter(*c)) {
                                    if (!tokenizer.eof())
                                        tokenizer.pushChar();
                                    return Token(TokenType::Decimal, begin, c, tokenizer.offset(begin), line, column);
                                }
                            }
                        }
                        
                        // read a word
                        while (!tokenizer.eof() && !isDelimiter(*(c = tokenizer.nextChar())));
                        if (!tokenizer.eof())
                            tokenizer.pushChar();
                        return Token(TokenType::String, begin, c, tokenizer.offset(begin), line, column);
                    }
                }
            }
            return Token(TokenType::Eof, NULL, NULL, 0, tokenizer.line(), tokenizer.column());
        }
开发者ID:rebb,项目名称:TrenchBroom,代码行数:84,代码来源:MapParser.cpp

示例2: doEmit

        Token DefTokenEmitter::doEmit(Tokenizer& tokenizer) {
            while (!tokenizer.eof()) {
                size_t line = tokenizer.line();
                size_t column = tokenizer.column();
                const char* c = tokenizer.nextChar();
                switch (*c) {
                    case '/': {
                        const char* begin = c;
                        if (tokenizer.peekChar() == '*') {
                            // eat all chars immediately after the '*' because it's often followed by QUAKE
                            while (!isWhitespace(*tokenizer.nextChar()) && !tokenizer.eof());
                            return Token(ODefinition, begin, c, tokenizer.offset(begin), line, column);
                        } else if (tokenizer.peekChar() == '/') {
                            // eat everything up to and including the next newline
                            while (*tokenizer.nextChar() != '\n');
                            break;
                        }
                        error(line, column, *c);
                        break;
                    }
                    case '*': {
                        const char* begin = c;
                        if (tokenizer.peekChar() == '/') {
                            tokenizer.nextChar();
                            return Token(ODefinition, begin, c, tokenizer.offset(begin), line, column);
                        }
                        error(line, column, *c);
                        break;
                    }
                    case '(':
                        return Token(OParenthesis, c, c + 1, tokenizer.offset(c), line, column);
                    case ')':
                        return Token(CParenthesis, c, c + 1, tokenizer.offset(c), line, column);
                    case '{':
                        return Token(OBrace, c, c + 1, tokenizer.offset(c), line, column);
                    case '}':
                        return Token(CBrace, c, c + 1, tokenizer.offset(c), line, column);
                    case '=':
                        return Token(Equality, c, c + 1, tokenizer.offset(c), line, column);
                    case ';':
                        return Token(Semicolon, c, c + 1, tokenizer.offset(c), line, column);
                    case '?':
                        return Token(Question, c, c + 1, tokenizer.offset(c), line, column);
                    case '\r':
                        if (tokenizer.peekChar() == '\n') {
                            tokenizer.nextChar();
                        }
                    case '\n':
                        return Token(Newline, c, c + 1, tokenizer.offset(c), line, column);
                    case ',':
                        return Token(Comma, c, c + 1, tokenizer.offset(c), line, column);
                    case ' ':
                    case '\t':
                        break;
                    case '"': { // quoted string
                        const char* begin = c;
                        const char* end;
                        tokenizer.quotedString(begin, end);
                        return Token(QuotedString, begin, end, tokenizer.offset(begin), line, column);
                    }
                    default: { // integer, decimal or word
                        const char* begin = c;

                        // try to read a number
                        if (*c == '-' || isDigit(*c)) {
                            while (isDigit(*(c = tokenizer.nextChar())));
                            if (isDelimiter(*c)) {
                                if (!tokenizer.eof())
                                    tokenizer.pushChar();
                                return Token(Integer, begin, c, tokenizer.offset(begin), line, column);
                            }
                        }
                        
                        // try to read a decimal (may start with '.')
                        if (*c == '.') {
                            while (isDigit(*(c = tokenizer.nextChar())));
                            if (isDelimiter(*c)) {
                                if (!tokenizer.eof())
                                    tokenizer.pushChar();
                                return Token(Decimal, begin, c, tokenizer.offset(begin), line, column);
                            }
                        }
                        
                        // read a word
                        while (!tokenizer.eof() && !isDelimiter(*(c = tokenizer.nextChar())));
                        if (!tokenizer.eof())
                            tokenizer.pushChar();
                        return Token(Word, begin, c, tokenizer.offset(begin), line, column);
                    }
                }
            }
            
            return Token(Eof, NULL, NULL, 0, tokenizer.line(), tokenizer.column());
        }
开发者ID:rebb,项目名称:TrenchBroom,代码行数:94,代码来源:DefParser.cpp

示例3: doEmit

 Token MapTokenEmitter::doEmit(Tokenizer& tokenizer, size_t line, size_t column) {
     const size_t startPosition = tokenizer.position();
     while (!tokenizer.eof()) {
         char c = tokenizer.nextChar();
         switch (c) {
             case '/':
                 if (tokenizer.peekChar() == '/') {
                     // eat everything up to and including the next newline
                     while (tokenizer.nextChar() != '\n');
                 }
                 break;
             case '{':
                 return Token(TokenType::OBrace, "", startPosition, tokenizer.position() - startPosition, line, column);
             case '}':
                 return Token(TokenType::CBrace, "", startPosition, tokenizer.position() - startPosition, line, column);
             case '(':
                 return Token(TokenType::OParenthesis, "", startPosition, tokenizer.position() - startPosition, line, column);
             case ')':
                 return Token(TokenType::CParenthesis, "", startPosition, tokenizer.position() - startPosition, line, column);
             case '[':
                 return Token(TokenType::OBracket, "", startPosition, tokenizer.position() - startPosition, line, column);
             case ']':
                 return Token(TokenType::CBracket, "", startPosition, tokenizer.position() - startPosition, line, column);
             case '"': // quoted string
                 m_buffer.str(String());
                 while (!tokenizer.eof() && (c = tokenizer.nextChar()) != '"')
                     m_buffer << c;
                 return Token(TokenType::String, m_buffer.str(), startPosition, tokenizer.position() - startPosition, line, column);
             default: // whitespace, integer, decimal or word
                 if (isWhitespace(c))
                     break;
                 
                 // clear the buffer
                 m_buffer.str(String());
                 
                 // try to read a number
                 if (c == '-' || isDigit(c)) {
                     m_buffer << c;
                     while (isDigit((c = tokenizer.nextChar())))
                         m_buffer << c;
                     if (isDelimiter(c)) {
                         if (!tokenizer.eof())
                             tokenizer.pushChar();
                         return Token(TokenType::Integer, m_buffer.str(), startPosition, tokenizer.position() - startPosition, line, column);
                     }
                 }
                 
                 // try to read a decimal (may start with '.')
                 if (c == '.') {
                     m_buffer << c;
                     while (isDigit((c = tokenizer.nextChar())))
                         m_buffer << c;
                     if (isDelimiter(c)) {
                         if (!tokenizer.eof())
                             tokenizer.pushChar();
                         return Token(TokenType::Decimal, m_buffer.str(), startPosition, tokenizer.position() - startPosition, line, column);
                     }
                 }
                 
                 // read a word
                 m_buffer << c;
                 while (!tokenizer.eof() && !isDelimiter(c = tokenizer.nextChar()))
                     m_buffer << c;
                 if (!tokenizer.eof())
                     tokenizer.pushChar();
                 return Token(TokenType::String, m_buffer.str(), startPosition, tokenizer.position() - startPosition, line, column);
         }
     }
     return Token(TokenType::Eof, "", startPosition, tokenizer.position() - startPosition, line, column);
 }
开发者ID:JoshEngebretson,项目名称:FSRadQuakeStuff,代码行数:70,代码来源:MapParser.cpp


注:本文中的Tokenizer::eof方法示例由纯净天空整理自Github/MSDocs等开源代码及文档管理平台,相关代码片段筛选自各路编程大神贡献的开源项目,源码版权归原作者所有,传播和使用请参考对应项目的License;未经允许,请勿转载。