C++ FormatToken类代码示例

/**
 * @brief Highlights rest of line as import directive
 */
void PythonHighlighter::highlightImport(Scanner &scanner)
{
    FormatToken tk;
    while ((tk = scanner.read()).format() != Format_EndOfBlock) {
        Format format = tk.format();
        if (tk.format() == Format_Identifier)
            format = Format_ImportedModule;
        setFormat(tk.begin(), tk.length(), formatForCategory(format));
    }
}

int GoHighlighter::highlightLine(const QString &text, int initialState)
{
    GoScanner scanner(text.constData(), text.size());
    scanner.setState(initialState);

    FormatToken tk;
    while ((tk = scanner.read()).format() != Format_EndOfBlock)
        setFormat(tk.begin(), tk.length(), formatForCategory(tk.format()));
    return scanner.state();
}

void Highlighter::highlightImport(Scanner &scanner)
{
    FormatToken tk;
    while ((tk = scanner.read()).format() != Format_EndOfBlock)
    {
        Format format = tk.format();
        if (tk.format() == Format_IDENTIFIER)
            format = Format_IMPORTED_MODULE;
        setFormat(tk.begin(), tk.length(), _formats[format]);
    }
}

int PythonIndenter::getIndentDiff(const QString &previousLine, const TextEditor::TabSettings &tabSettings) const
{
    PythonScanner sc(previousLine.constData(), previousLine.length());
        forever {
            FormatToken tk = sc.read();
            if ((tk.format() == Internal::Format_Keyword) && m_jumpKeywords.contains(sc.value(tk)))
                return -tabSettings.m_indentSize;
            if (tk.format() != Internal::Format_Whitespace)
                break;
        }
        return 0;
}

void WhitespaceManager::breakToken(const FormatToken &Tok, unsigned Offset,
                                   unsigned ReplaceChars,
                                   StringRef PreviousPostfix,
                                   StringRef CurrentPrefix, bool InPPDirective,
                                   unsigned Spaces) {
  Changes.push_back(Change(
      true, SourceRange(Tok.getStartOfNonWhitespace().getLocWithOffset(Offset),
                        Tok.getStartOfNonWhitespace().getLocWithOffset(
                            Offset + ReplaceChars)),
      Spaces, Spaces, 1, PreviousPostfix, CurrentPrefix,
      // FIXME: Unify token adjustment, so we don't split it between
      // BreakableToken and the WhitespaceManager. That would also allow us to
      // correctly store a tok::TokenKind instead of rolling our own enum.
      tok::unknown, InPPDirective && !Tok.IsFirst));
}

void WhitespaceManager::replaceWhitespaceInToken(
    const FormatToken &Tok, unsigned Offset, unsigned ReplaceChars,
    StringRef PreviousPostfix, StringRef CurrentPrefix, bool InPPDirective,
    unsigned Newlines, unsigned Spaces) {
  Changes.push_back(Change(
      true, SourceRange(Tok.getStartOfNonWhitespace().getLocWithOffset(Offset),
                        Tok.getStartOfNonWhitespace().getLocWithOffset(
                            Offset + ReplaceChars)),
      Spaces, Spaces, Newlines, PreviousPostfix, CurrentPrefix,
      // If we don't add a newline this change doesn't start a comment. Thus,
      // when we align line comments, we don't need to treat this change as one.
      // FIXME: We still need to take this change in account to properly
      // calculate the new length of the comment and to calculate the changes
      // for which to do the alignment when aligning comments.
      Tok.Type == TT_LineComment && Newlines > 0 ? tok::comment : tok::unknown,
      InPPDirective && !Tok.IsFirst));
}

bool AffectedRangeManager::affectsTokenRange(const FormatToken &First,
                                             const FormatToken &Last,
                                             bool IncludeLeadingNewlines) {
  SourceLocation Start = First.WhitespaceRange.getBegin();
  if (!IncludeLeadingNewlines)
    Start = Start.getLocWithOffset(First.LastNewlineOffset);
  SourceLocation End = Last.getStartOfNonWhitespace();
  End = End.getLocWithOffset(Last.TokenText.size());
  CharSourceRange Range = CharSourceRange::getCharRange(Start, End);
  return affectsCharSourceRange(Range);
}

int LuaHighlighter::highlightLine(QString const& text, int initialState)
{
	Scanner scanner(text.constData(),text.length());
	scanner.setState(initialState);
	
	FormatToken tk;
	bool hasOnlyWhitespace = true;
	while((tk = scanner.read()).format() != Format_EndOfBlock)
	{
		Format format = tk.format();
		if(format == Format_Keyword) {
			if(isImportKeyword(scanner.value(tk)) && hasOnlyWhitespace)
			{
				setFormat(tk.begin(), tk.length(), formatForCategory(format));
				highlightImport(scanner);
				break;
			}
		}
		
		setFormat(tk.begin(), tk.length(), formatForCategory(format));
		if(format != Format_Whitespace)
			hasOnlyWhitespace = false;
	}
	return scanner.state();
}

void UnwrappedLineFormatter::formatFirstToken(FormatToken &RootToken,
                                              const AnnotatedLine *PreviousLine,
                                              unsigned IndentLevel,
                                              unsigned Indent,
                                              bool InPPDirective) {
  unsigned Newlines =
      std::min(RootToken.NewlinesBefore, Style.MaxEmptyLinesToKeep + 1);
  // Remove empty lines before "}" where applicable.
  if (RootToken.is(tok::r_brace) &&
      (!RootToken.Next ||
       (RootToken.Next->is(tok::semi) && !RootToken.Next->Next)))
    Newlines = std::min(Newlines, 1u);
  if (Newlines == 0 && !RootToken.IsFirst)
    Newlines = 1;
  if (RootToken.IsFirst && !RootToken.HasUnescapedNewline)
    Newlines = 0;

  // Remove empty lines after "{".
  if (!Style.KeepEmptyLinesAtTheStartOfBlocks && PreviousLine &&
      PreviousLine->Last->is(tok::l_brace) &&
      PreviousLine->First->isNot(tok::kw_namespace) &&
      !startsExternCBlock(*PreviousLine))
    Newlines = 1;

  // Insert extra new line before access specifiers.
  if (PreviousLine && PreviousLine->Last->isOneOf(tok::semi, tok::r_brace) &&
      RootToken.isAccessSpecifier() && RootToken.NewlinesBefore == 1)
    ++Newlines;

  // Remove empty lines after access specifiers.
  if (PreviousLine && PreviousLine->First->isAccessSpecifier() &&
      (!PreviousLine->InPPDirective || !RootToken.HasUnescapedNewline))
    Newlines = std::min(1u, Newlines);

  Whitespaces->replaceWhitespace(RootToken, Newlines, IndentLevel, Indent,
                                 Indent, InPPDirective &&
                                             !RootToken.HasUnescapedNewline);
}

void UnwrappedLineParser::calculateBraceTypes() {
  // We'll parse forward through the tokens until we hit
  // a closing brace or eof - note that getNextToken() will
  // parse macros, so this will magically work inside macro
  // definitions, too.
  unsigned StoredPosition = Tokens->getPosition();
  unsigned Position = StoredPosition;
  FormatToken *Tok = FormatTok;
  // Keep a stack of positions of lbrace tokens. We will
  // update information about whether an lbrace starts a
  // braced init list or a different block during the loop.
  SmallVector<FormatToken *, 8> LBraceStack;
  assert(Tok->Tok.is(tok::l_brace));
  do {
    // Get next none-comment token.
    FormatToken *NextTok;
    unsigned ReadTokens = 0;
    do {
      NextTok = Tokens->getNextToken();
      ++ReadTokens;
    } while (NextTok->is(tok::comment));

    switch (Tok->Tok.getKind()) {
    case tok::l_brace:
      LBraceStack.push_back(Tok);
      break;
    case tok::r_brace:
      if (!LBraceStack.empty()) {
        if (LBraceStack.back()->BlockKind == BK_Unknown) {
          bool ProbablyBracedList = false;
          if (Style.Language == FormatStyle::LK_Proto) {
            ProbablyBracedList = NextTok->isOneOf(tok::comma, tok::r_square);
          } else {
            // Using OriginalColumn to distinguish between ObjC methods and
            // binary operators is a bit hacky.
            bool NextIsObjCMethod = NextTok->isOneOf(tok::plus, tok::minus) &&
                                    NextTok->OriginalColumn == 0;

            // If there is a comma, semicolon or right paren after the closing
            // brace, we assume this is a braced initializer list.  Note that
            // regardless how we mark inner braces here, we will overwrite the
            // BlockKind later if we parse a braced list (where all blocks
            // inside are by default braced lists), or when we explicitly detect
            // blocks (for example while parsing lambdas).
            //
            // We exclude + and - as they can be ObjC visibility modifiers.
            ProbablyBracedList =
                NextTok->isOneOf(tok::comma, tok::semi, tok::period, tok::colon,
                                 tok::r_paren, tok::r_square, tok::l_brace,
                                 tok::l_paren) ||
                (NextTok->isBinaryOperator() && !NextIsObjCMethod);
          }
          if (ProbablyBracedList) {
            Tok->BlockKind = BK_BracedInit;
            LBraceStack.back()->BlockKind = BK_BracedInit;
          } else {
            Tok->BlockKind = BK_Block;
            LBraceStack.back()->BlockKind = BK_Block;
          }
        }
        LBraceStack.pop_back();
      }
      break;
    case tok::at:
    case tok::semi:
    case tok::kw_if:
    case tok::kw_while:
    case tok::kw_for:
    case tok::kw_switch:
    case tok::kw_try:
      if (!LBraceStack.empty())
        LBraceStack.back()->BlockKind = BK_Block;
      break;
    default:
      break;
    }
    Tok = NextTok;
    Position += ReadTokens;
  } while (Tok->Tok.isNot(tok::eof) && !LBraceStack.empty());
  // Assume other blocks for all unclosed opening braces.
  for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) {
    if (LBraceStack[i]->BlockKind == BK_Unknown)
      LBraceStack[i]->BlockKind = BK_Block;
  }

  FormatTok = Tokens->setPosition(StoredPosition);
}

void CommaSeparatedList::precomputeFormattingInfos(const FormatToken *Token) {
  // FIXME: At some point we might want to do this for other lists, too.
  if (!Token->MatchingParen || Token->isNot(tok::l_brace))
    return;

  FormatToken *ItemBegin = Token->Next;
  SmallVector<bool, 8> MustBreakBeforeItem;

  // The lengths of an item if it is put at the end of the line. This includes
  // trailing comments which are otherwise ignored for column alignment.
  SmallVector<unsigned, 8> EndOfLineItemLength;

  for (unsigned i = 0, e = Commas.size() + 1; i != e; ++i) {
    // Skip comments on their own line.
    while (ItemBegin->HasUnescapedNewline && ItemBegin->isTrailingComment())
      ItemBegin = ItemBegin->Next;

    MustBreakBeforeItem.push_back(ItemBegin->MustBreakBefore);
    const FormatToken *ItemEnd = NULL;
    if (i == Commas.size()) {
      ItemEnd = Token->MatchingParen;
      const FormatToken *NonCommentEnd = ItemEnd->getPreviousNonComment();
      ItemLengths.push_back(CodePointsBetween(ItemBegin, NonCommentEnd));
      if (Style.Cpp11BracedListStyle) {
        // In Cpp11 braced list style, the } and possibly other subsequent
        // tokens will need to stay on a line with the last element.
        while (ItemEnd->Next && !ItemEnd->Next->CanBreakBefore)
          ItemEnd = ItemEnd->Next;
      } else {
        // In other braced lists styles, the "}" can be wrapped to the new line.
        ItemEnd = Token->MatchingParen->Previous;
      }
    } else {
      ItemEnd = Commas[i];
      // The comma is counted as part of the item when calculating the length.
      ItemLengths.push_back(CodePointsBetween(ItemBegin, ItemEnd));
      // Consume trailing comments so the are included in EndOfLineItemLength.
      if (ItemEnd->Next && !ItemEnd->Next->HasUnescapedNewline &&
          ItemEnd->Next->isTrailingComment())
        ItemEnd = ItemEnd->Next;
    }
    EndOfLineItemLength.push_back(CodePointsBetween(ItemBegin, ItemEnd));
    // If there is a trailing comma in the list, the next item will start at the
    // closing brace. Don't create an extra item for this.
    if (ItemEnd->getNextNonComment() == Token->MatchingParen)
      break;
    ItemBegin = ItemEnd->Next;
  }

  // We can never place more than ColumnLimit / 3 items in a row (because of the
  // spaces and the comma).
  for (unsigned Columns = 1; Columns <= Style.ColumnLimit / 3; ++Columns) {
    ColumnFormat Format;
    Format.Columns = Columns;
    Format.ColumnSizes.resize(Columns);
    Format.LineCount = 0;
    bool HasRowWithSufficientColumns = false;
    unsigned Column = 0;
    for (unsigned i = 0, e = ItemLengths.size(); i != e; ++i) {
      if (MustBreakBeforeItem[i] || Column == Columns) {
        ++Format.LineCount;
        Column = 0;
      }
      if (Column == Columns - 1)
        HasRowWithSufficientColumns = true;
      unsigned length =
          (Column == Columns - 1) ? EndOfLineItemLength[i] : ItemLengths[i];
      Format.ColumnSizes[Column] =
          std::max(Format.ColumnSizes[Column], length);
      ++Column;
    }
    // If all rows are terminated early (e.g. by trailing comments), we don't
    // need to look further.
    if (!HasRowWithSufficientColumns)
      break;
    Format.TotalWidth = Columns - 1; // Width of the N-1 spaces.
    for (unsigned i = 0; i < Columns; ++i) {
      Format.TotalWidth += Format.ColumnSizes[i];
    }

    // Ignore layouts that are bound to violate the column limit.
    if (Format.TotalWidth > Style.ColumnLimit)
      continue;

    Formats.push_back(Format);
  }
}

DebugCommand* PrintCommand::Build(std::vector<Token*> args) {
  if (args.size() < 2) {
    return new InvalidCommand(args, -1, "too few arguments");
  }

  Token* target = args[1];
  if (!target->IsRegister() &&
      !target->IsFPRegister() &&
      !target->IsIdentifier()) {
    return new InvalidCommand(args, 1, "expects reg or identifier");
  }

  FormatToken* format = NULL;
  int target_size = 0;
  if (target->IsRegister()) {
    Register reg = RegisterToken::Cast(target)->value();
    target_size = reg.SizeInBytes();
  } else if (target->IsFPRegister()) {
    FPRegister fpreg = FPRegisterToken::Cast(target)->value();
    target_size = fpreg.SizeInBytes();
  }
  // If the target is an identifier there must be no format. This is checked
  // in the switch statement below.

  switch (args.size()) {
    case 2: {
      if (target->IsRegister()) {
        switch (target_size) {
          case 4: format = new Format<uint32_t>("%08" PRIx32, 'x'); break;
          case 8: format = new Format<uint64_t>("%016" PRIx64, 'x'); break;
          default: VIXL_UNREACHABLE();
        }
      } else if (target->IsFPRegister()) {
        switch (target_size) {
          case 4: format = new Format<float>("%8g", 'f'); break;
          case 8: format = new Format<double>("%8g", 'f'); break;
          default: VIXL_UNREACHABLE();
        }
      }
      break;
    }
    case 3: {
      if (target->IsIdentifier()) {
        return new InvalidCommand(args, 2,
            "format is only allowed with registers");
      }

      Token* second = args[2];
      if (!second->IsFormat()) {
        return new InvalidCommand(args, 2, "expects format");
      }
      format = FormatToken::Cast(second);

      if (format->SizeOf() > target_size) {
        return new InvalidCommand(args, 2, "format too wide");
      }

      break;
    }
    default:
      return new InvalidCommand(args, -1, "too many arguments");
  }

  return new PrintCommand(args[0], target, format);
}

void CommaSeparatedList::precomputeFormattingInfos(const FormatToken *Token) {
  // FIXME: At some point we might want to do this for other lists, too.
  if (!Token->MatchingParen || Token->isNot(tok::l_brace))
    return;

  // In C++11 braced list style, we should not format in columns unless they
  // have many items (20 or more) or we allow bin-packing of function call
  // arguments.
  if (Style.Cpp11BracedListStyle && !Style.BinPackArguments &&
      Commas.size() < 19)
    return;

  // Column format doesn't really make sense if we don't align after brackets.
  if (!Style.AlignAfterOpenBracket)
    return;

  FormatToken *ItemBegin = Token->Next;
  while (ItemBegin->isTrailingComment())
    ItemBegin = ItemBegin->Next;
  SmallVector<bool, 8> MustBreakBeforeItem;

  // The lengths of an item if it is put at the end of the line. This includes
  // trailing comments which are otherwise ignored for column alignment.
  SmallVector<unsigned, 8> EndOfLineItemLength;

  bool HasSeparatingComment = false;
  for (unsigned i = 0, e = Commas.size() + 1; i != e; ++i) {
    // Skip comments on their own line.
    while (ItemBegin->HasUnescapedNewline && ItemBegin->isTrailingComment()) {
      ItemBegin = ItemBegin->Next;
      HasSeparatingComment = i > 0;
    }

    MustBreakBeforeItem.push_back(ItemBegin->MustBreakBefore);
    if (ItemBegin->is(tok::l_brace))
      HasNestedBracedList = true;
    const FormatToken *ItemEnd = nullptr;
    if (i == Commas.size()) {
      ItemEnd = Token->MatchingParen;
      const FormatToken *NonCommentEnd = ItemEnd->getPreviousNonComment();
      ItemLengths.push_back(CodePointsBetween(ItemBegin, NonCommentEnd));
      if (Style.Cpp11BracedListStyle) {
        // In Cpp11 braced list style, the } and possibly other subsequent
        // tokens will need to stay on a line with the last element.
        while (ItemEnd->Next && !ItemEnd->Next->CanBreakBefore)
          ItemEnd = ItemEnd->Next;
      } else {
        // In other braced lists styles, the "}" can be wrapped to the new line.
        ItemEnd = Token->MatchingParen->Previous;
      }
    } else {
      ItemEnd = Commas[i];
      // The comma is counted as part of the item when calculating the length.
      ItemLengths.push_back(CodePointsBetween(ItemBegin, ItemEnd));

      // Consume trailing comments so the are included in EndOfLineItemLength.
      if (ItemEnd->Next && !ItemEnd->Next->HasUnescapedNewline &&
          ItemEnd->Next->isTrailingComment())
        ItemEnd = ItemEnd->Next;
    }
    EndOfLineItemLength.push_back(CodePointsBetween(ItemBegin, ItemEnd));
    // If there is a trailing comma in the list, the next item will start at the
    // closing brace. Don't create an extra item for this.
    if (ItemEnd->getNextNonComment() == Token->MatchingParen)
      break;
    ItemBegin = ItemEnd->Next;
  }

  // Don't use column layout for nested lists, lists with few elements and in
  // presence of separating comments.
  if (Token->NestingLevel != 0 || Commas.size() < 5 || HasSeparatingComment)
    return;

  // We can never place more than ColumnLimit / 3 items in a row (because of the
  // spaces and the comma).
  unsigned MaxItems = Style.ColumnLimit / 3;
  std::vector<unsigned> MinSizeInColumn;
  MinSizeInColumn.reserve(MaxItems);
  for (unsigned Columns = 1; Columns <= MaxItems; ++Columns) {
    ColumnFormat Format;
    Format.Columns = Columns;
    Format.ColumnSizes.resize(Columns);
    MinSizeInColumn.assign(Columns, UINT_MAX);
    Format.LineCount = 1;
    bool HasRowWithSufficientColumns = false;
    unsigned Column = 0;
    for (unsigned i = 0, e = ItemLengths.size(); i != e; ++i) {
      assert(i < MustBreakBeforeItem.size());
      if (MustBreakBeforeItem[i] || Column == Columns) {
        ++Format.LineCount;
        Column = 0;
      }
      if (Column == Columns - 1)
        HasRowWithSufficientColumns = true;
      unsigned Length =
          (Column == Columns - 1) ? EndOfLineItemLength[i] : ItemLengths[i];
      Format.ColumnSizes[Column] = std::max(Format.ColumnSizes[Column], Length);
      MinSizeInColumn[Column] = std::min(MinSizeInColumn[Column], Length);
      ++Column;
    }
//.........这里部分代码省略.........

void UnwrappedLineParser::calculateBraceTypes() {
  // We'll parse forward through the tokens until we hit
  // a closing brace or eof - note that getNextToken() will
  // parse macros, so this will magically work inside macro
  // definitions, too.
  unsigned StoredPosition = Tokens->getPosition();
  unsigned Position = StoredPosition;
  FormatToken *Tok = FormatTok;
  // Keep a stack of positions of lbrace tokens. We will
  // update information about whether an lbrace starts a
  // braced init list or a different block during the loop.
  SmallVector<FormatToken *, 8> LBraceStack;
  assert(Tok->Tok.is(tok::l_brace));
  do {
    // Get next none-comment token.
    FormatToken *NextTok;
    unsigned ReadTokens = 0;
    do {
      NextTok = Tokens->getNextToken();
      ++ReadTokens;
    } while (NextTok->is(tok::comment));

    switch (Tok->Tok.getKind()) {
    case tok::l_brace:
      LBraceStack.push_back(Tok);
      break;
    case tok::r_brace:
      if (!LBraceStack.empty()) {
        if (LBraceStack.back()->BlockKind == BK_Unknown) {
          // If there is a comma, semicolon or right paren after the closing
          // brace, we assume this is a braced initializer list.

          // FIXME: Note that this currently works only because we do not
          // use the brace information while inside a braced init list.
          // Thus, if the parent is a braced init list, we consider all
          // brace blocks inside it braced init list. That works good enough
          // for now, but we will need to fix it to correctly handle lambdas.
          if (NextTok->isOneOf(tok::comma, tok::semi, tok::r_paren,
                               tok::l_brace, tok::colon)) {
            Tok->BlockKind = BK_BracedInit;
            LBraceStack.back()->BlockKind = BK_BracedInit;
          } else {
            Tok->BlockKind = BK_Block;
            LBraceStack.back()->BlockKind = BK_Block;
          }
        }
        LBraceStack.pop_back();
      }
      break;
    case tok::semi:
    case tok::kw_if:
    case tok::kw_while:
    case tok::kw_for:
    case tok::kw_switch:
    case tok::kw_try:
      if (!LBraceStack.empty())
        LBraceStack.back()->BlockKind = BK_Block;
      break;
    default:
      break;
    }
    Tok = NextTok;
    Position += ReadTokens;
  } while (Tok->Tok.isNot(tok::eof));
  // Assume other blocks for all unclosed opening braces.
  for (unsigned i = 0, e = LBraceStack.size(); i != e; ++i) {
    if (LBraceStack[i]->BlockKind == BK_Unknown)
      LBraceStack[i]->BlockKind = BK_Block;
  }
  FormatTok = Tokens->setPosition(StoredPosition);
}

/**
 * @brief Highlighter::highlight_impl
 * @param text - source code to highlight
 * @param initialState - initial state of scanner, retrieved from previous block
 * @return final state of scanner, should be saved with current block
 */
int Highlighter::highlightLine(const QString &text, int initialState)
{
    Scanner scanner(text.constData(), text.size());
    scanner.setState(initialState);

    HState hstate = HState_NORMAL;

    int blockNo = currentBlock().blockNumber();
    QMap<int, QString>::iterator it = _userTypes.find(blockNo);
    if (it != _userTypes.end())
        _userTypes.erase(it);

    FormatToken tk;
    bool hasOnlyWhitespace = true;
    while ((tk = scanner.read()).format() != Format_EndOfBlock)
    {
        Format format = tk.format();
        switch (hstate)
        {
        default: // HState_NORMAL
            if (format == Format_KEYWORD) {
                QString value = scanner.value(tk);
                if (value == "class") {
                    hstate = HState_ON_CLASS;
                } else if (isImportKeyword(value) && hasOnlyWhitespace) {
                    setFormat(tk.begin(), tk.length(), _formats[format]);
                    highlightImport(scanner);
                    return scanner.getState();
                }
            }
            if (format == Format_IDENTIFIER) {
                QString value = scanner.value(tk);
                QMap<int, QString>::const_iterator it;
                for (it = _userTypes.begin(); it != _userTypes.end(); ++it) {
                    if (value == it.value()) {
                        format = Format_TYPE;
                        break;
                    }
                }
            }
            break;

        case HState_ON_CLASS:
            if (format == Format_WHITESPACE)
                hstate = HState_AFTER_CLASS;
            else
                hstate = HState_NORMAL;
            break;

        case HState_AFTER_CLASS:
            hstate = HState_NORMAL;
            if (format == Format_IDENTIFIER) {
                _userTypes[blockNo] = scanner.value(tk);
                format = Format_TYPE;
            }
            break;
        }
        setFormat(tk.begin(), tk.length(), _formats[format]);
        if (format != Format_WHITESPACE)
            hasOnlyWhitespace = false;
    }
    return scanner.getState();
}