C++ tok_ctx::get方法代码示例

/**
 * Literal string, ends with single "
 * Two "" don't end the string.
 *
 * @param pc   The structure to update, str is an input.
 * @return     Whether a string was parsed
 */
static bool parse_cs_string(tok_ctx& ctx, chunk_t& pc)
{
   pc.str = ctx.get();
   pc.str.append(ctx.get());

   /* go until we hit a zero (end of file) or a single " */
   while (ctx.more())
   {
      int ch = ctx.get();
      pc.str.append(ch);
      if (ch == '"')
      {
         if (ctx.peek() == '"')
         {
            pc.str.append(ctx.get());
         }
         else
         {
            break;
         }
      }
   }

   pc.type = CT_STRING;
   return(true);
}

/**
 * Count the number of characters in a quoted string.
 * The next bit of text starts with a quote char " or ' or <.
 * Count the number of characters until the matching character.
 *
 * @param pc   The structure to update, str is an input.
 * @return     Whether a string was parsed
 */
static bool parse_string(tok_ctx& ctx, chunk_t& pc, int quote_idx, bool allow_escape)
{
   bool escaped = 0;
   int  end_ch;
   char escape_char  = cpd.settings[UO_string_escape_char].n;
   char escape_char2 = cpd.settings[UO_string_escape_char2].n;

   pc.str.clear();
   while (quote_idx-- > 0)
   {
      pc.str.append(ctx.get());
   }

   pc.type = CT_STRING;
   end_ch  = CharTable::Get(ctx.peek()) & 0xff;
   pc.str.append(ctx.get());  /* store the " */

   while (ctx.more())
   {
      int ch = ctx.get();
      pc.str.append(ch);
      if (ch == '\n')
      {
         pc.nl_count++;
         pc.type = CT_STRING_MULTI;
         escaped = 0;
         continue;
      }
      if ((ch == '\r') && (ctx.peek() != '\n'))
      {
         pc.str.append(ctx.get());
         pc.nl_count++;
         pc.type = CT_STRING_MULTI;
         escaped = 0;
         continue;
      }
      if (!escaped)
      {
         if (ch == escape_char)
         {
            escaped = (escape_char != 0);
         }
         else if ((ch == escape_char2) && (ctx.peek() == end_ch))
         {
            escaped = allow_escape;
         }
         else if (ch == end_ch)
         {
            break;
         }
      }
      else
      {
         escaped = false;
      }
   }

   parse_suffix(ctx, pc, true);
   return(true);
}

/**
 * Parses any number of tab or space chars followed by a newline.
 * Does not change pc.len if a newline isn't found.
 * This is not the same as parse_whitespace() because it only consumes until
 * a single newline is encountered.
 */
static bool parse_newline(tok_ctx& ctx)
{
   ctx.save();

   /* Eat whitespace */
   while ((ctx.peek() == ' ') || (ctx.peek() == '\t'))
   {
      ctx.get();
   }
   if ((ctx.peek() == '\r') || (ctx.peek() == '\n'))
   {
      if (ctx.peek() == '\n')
      {
         ctx.get();
      }
      else /* it is '\r' */
      {
         ctx.get();
         if (ctx.peek() == '\n')
         {
            ctx.get();
         }
      }
      return(true);
   }
   ctx.restore();
   return(false);
}

/**
 * Called when we hit a backslash.
 * If there is nothing but whitespace until the newline, then this is a
 * backslash newline
 */
static bool parse_bs_newline(tok_ctx& ctx, chunk_t& pc)
{
   ctx.save();
   ctx.get(); /* skip the '\' */

   int ch;
   while (ctx.more() && unc_isspace(ch = ctx.peek()))
   {
      ctx.get();
      if ((ch == '\r') || (ch == '\n'))
      {
         if (ch == '\r')
         {
            ctx.expect('\n');
         }
         pc.str      = "\\";
         pc.type     = CT_NL_CONT;
         pc.nl_count = 1;
         return(true);
      }
   }

   ctx.restore();
   return(false);
}

/**
 * VALA verbatim string, ends with three quotes (""")
 *
 * @param pc   The structure to update, str is an input.
 */
static void parse_verbatim_string(tok_ctx& ctx, chunk_t& pc)
{
   pc.type = CT_STRING;

   // consume the initial """
   pc.str = ctx.get();
   pc.str.append(ctx.get());
   pc.str.append(ctx.get());

   /* go until we hit a zero (end of file) or a """ */
   while (ctx.more())
   {
      int ch = ctx.get();
      pc.str.append(ch);
      if ((ch == '"') &&
          (ctx.peek() == '"') &&
          (ctx.peek(1) == '"'))
      {
         pc.str.append(ctx.get());
         pc.str.append(ctx.get());
         break;
      }
      if ((ch == '\n') || (ch == '\r'))
      {
         pc.type = CT_STRING_MULTI;
         pc.nl_count++;
      }
   }
}

/**
 * Figure of the length of the code placeholder at text, if present.
 * This is only for Xcode which sometimes inserts temporary code placeholder chunks, which in plaintext <#look like this#>.
 *
 * @param pc   The structure to update, str is an input.
 * @return     Whether a placeholder was parsed.
 */
static bool parse_code_placeholder(tok_ctx& ctx, chunk_t& pc)
{
   int last2 = 0, last1 = 0;

   if ((ctx.peek() != '<') || (ctx.peek(1) != '#'))
   {
      return(false);
   }

   ctx.save();

   /* account for opening two chars '<#' */
   pc.str = ctx.get();
   pc.str.append(ctx.get());

   /* grab everything until '#>', fail if not found. */
   while (ctx.more())
   {
      last2 = last1;
      last1 = ctx.get();
      pc.str.append(last1);

      if ((last2 == '#') && (last1 == '>'))
      {
         pc.type = CT_WORD;
         return(true);
      }
   }
   ctx.restore();
   return(false);
}

/**
 * Parses a C++0x 'R' string. R"( xxx )" R"tag(  )tag" u8R"(x)" uR"(x)"
 * Newlines may be in the string.
 */
static bool parse_cr_string(tok_ctx& ctx, chunk_t& pc, int q_idx)
{
   int cnt;
   int tag_idx = ctx.c.idx + q_idx + 1;
   int tag_len = 0;

   ctx.save();

   /* Copy the prefix + " to the string */
   pc.str.clear();
   cnt = q_idx + 1;
   while (cnt--)
   {
      pc.str.append(ctx.get());
   }

   /* Add the tag and get the length of the tag */
   while (ctx.more() && (ctx.peek() != '('))
   {
      tag_len++;
      pc.str.append(ctx.get());
   }
   if (ctx.peek() != '(')
   {
      ctx.restore();
      return(false);
   }

   pc.type = CT_STRING;
   while (ctx.more())
   {
      if ((ctx.peek() == ')') &&
          (ctx.peek(tag_len + 1) == '"') &&
          tag_compare(ctx.data, tag_idx, ctx.c.idx + 1, tag_len))
      {
         cnt = tag_len + 2;   /* for the )" */
         while (cnt--)
         {
            pc.str.append(ctx.get());
         }
         parse_suffix(ctx, pc);
         return(true);
      }
      if (ctx.peek() == '\n')
      {
         pc.str.append(ctx.get());
         pc.nl_count++;
         pc.type = CT_STRING_MULTI;
      }
      else
      {
         pc.str.append(ctx.get());
      }
   }
   ctx.restore();
   return(false);
} // parse_cr_string

/**
 * Count the number of characters in a word.
 * The first character is already valid for a keyword
 *
 * @param pc   The structure to update, str is an input.
 * @return     Whether a word was parsed (always true)
 */
bool parse_word(tok_ctx& ctx, chunk_t& pc, bool skipcheck)
{
   int             ch;
   static unc_text intr_txt("@interface");

   /* The first character is already valid */
   pc.str.clear();
   pc.str.append(ctx.get());

   while (ctx.more() && CharTable::IsKw2(ctx.peek()))
   {
      ch = ctx.get();
      pc.str.append(ch);

      /* HACK: Non-ASCII character are only allowed in identifiers */
      if (ch > 0x7f)
      {
         skipcheck = true;
      }
   }
   pc.type = CT_WORD;

   if (skipcheck)
   {
      return(true);
   }

   /* Detect pre-processor functions now */
   if ((cpd.in_preproc == CT_PP_DEFINE) &&
       (cpd.preproc_ncnl_count == 1))
   {
      if (ctx.peek() == '(')
      {
         pc.type = CT_MACRO_FUNC;
      }
      else
      {
         pc.type = CT_MACRO;
      }
   }
   else
   {
      /* '@interface' is reserved, not an interface itself */
      if ((cpd.lang_flags & LANG_JAVA) && pc.str.startswith("@") &&
          !pc.str.equals(intr_txt))
      {
         pc.type = CT_ANNOTATION;
      }
      else
      {
         /* Turn it into a keyword now */
         pc.type = find_keyword_type(pc.text(), pc.str.size());
      }
   }

   return(true);
} // parse_word

/**
 * Literal string, ends with single "
 * Two "" don't end the string.
 *
 * @param pc   The structure to update, str is an input.
 * @return     Whether a string was parsed
 */
static bool parse_cs_string(tok_ctx &ctx, chunk_t &pc)
{
   pc.str = ctx.get();
   pc.str.append(ctx.get());
   pc.type = CT_STRING;

   bool should_escape_tabs = cpd.settings[UO_string_replace_tab_chars].b;

   /* go until we hit a zero (end of file) or a single " */
   while (ctx.more())
   {
      int ch = ctx.get();
      pc.str.append(ch);
      if ((ch == '\n') || (ch == '\r'))
      {
         pc.type = CT_STRING_MULTI;
         pc.nl_count++;
      }
      else if (ch == '\t')
      {
         if (should_escape_tabs && !cpd.warned_unable_string_replace_tab_chars)
         {
            cpd.warned_unable_string_replace_tab_chars = true;

            log_sev_t warnlevel = (log_sev_t)cpd.settings[UO_warn_level_tabs_found_in_verbatim_string_literals].n;

            /* a tab char can't be replaced with \\t because escapes don't work in here-strings. best we can do is warn. */
            LOG_FMT(warnlevel, "%s:%d Detected non-replaceable tab char in literal string\n", cpd.filename, pc.orig_line);
            if (warnlevel < LWARN)
            {
               cpd.error_count++;
            }
         }
      }
      else if (ch == '"')
      {
         if (ctx.peek() == '"')
         {
            pc.str.append(ctx.get());
         }
         else
         {
            break;
         }
      }
   }

   return(true);
} // parse_cs_string

/**
 * Count the number of whitespace characters.
 *
 * @param pc   The structure to update, str is an input.
 * @return     Whether whitespace was parsed
 */
static bool parse_whitespace(tok_ctx& ctx, chunk_t& pc)
{
   int nl_count = 0;
   int ch       = -2;

   /* REVISIT: use a better whitespace detector? */
   while (ctx.more() && unc_isspace(ctx.peek()))
   {
      ch = ctx.get();   /* throw away the whitespace char */
      switch (ch)
      {
      case '\r':
         if (ctx.peek() == '\n')
         {
            /* CRLF ending */
            ctx.get();     /* throw away \n */
            cpd.le_counts[LE_CRLF]++;
         }
         else
         {
            /* CR ending */
            cpd.le_counts[LE_CR]++;
         }
         nl_count++;
         break;

      case '\n':
         /* LF ending */
         cpd.le_counts[LE_LF]++;
         nl_count++;
         break;

      case '\t':
      case ' ':
      default:
         break;
      }
   }

   if (ch != -2)
   {
      pc.str.clear();
      pc.nl_count  = nl_count;
      pc.type      = nl_count ? CT_NEWLINE : CT_WHITESPACE;
      pc.after_tab = (ctx.c.last_ch == '\t');
      return(true);
   }
   return(false);
}

/**
 * Parse any attached suffix, which may be a user-defined literal suffix.
 * If for a string, explicitly exclude common format and scan specifiers, ie,
 * PRIx32 and SCNx64.
 */
static void parse_suffix(tok_ctx& ctx, chunk_t& pc, bool forstring = false)
{
   if (CharTable::IsKw1(ctx.peek()))
   {
      int      slen    = 0;
      int      oldsize = pc.str.size();
      tok_info ss;

      /* don't add the suffix if we see L" or L' or S" */
      int p1 = ctx.peek();
      int p2 = ctx.peek(1);
      if (forstring &&
          (((p1 == 'L') && ((p2 == '"') || (p2 == '\''))) ||
           ((p1 == 'S') && (p2 == '"'))))
      {
         return;
      }
      ctx.save(ss);
      while (ctx.more() && CharTable::IsKw2(ctx.peek()))
      {
         slen++;
         pc.str.append(ctx.get());
      }

      if (forstring && (slen >= 4) &&
          (pc.str.startswith("PRI", oldsize) ||
           pc.str.startswith("SCN", oldsize)))
      {
         ctx.restore(ss);
         pc.str.resize(oldsize);
      }
   }
}

/**
 * Count the number of characters in a word.
 * The first character is already valid for a keyword
 *
 * @param pc   The structure to update, str is an input.
 * @return     Whether a word was parsed (always true)
 */
bool parse_word(tok_ctx& ctx, chunk_t& pc, bool skipcheck)
{
   int ch;

   /* The first character is already valid */
   pc.str.clear();
   pc.str.append(ctx.get());

   while (ctx.more() && CharTable::IsKw2(ctx.peek()))
   {
      ch = ctx.get();
      pc.str.append(ch);

      /* HACK: Non-ASCII character are only allowed in identifiers */
      if (ch > 0x7f)
      {
         skipcheck = true;
      }
   }
   pc.type = CT_WORD;

   if (skipcheck)
   {
      return(true);
   }

   /* Detect pre-processor functions now */
   if ((cpd.in_preproc == CT_PP_DEFINE) &&
       (cpd.preproc_ncnl_count == 1))
   {
      if (ctx.peek() == '(')
      {
         pc.type = CT_MACRO_FUNC;
      }
      else
      {
         pc.type = CT_MACRO;
      }
   }
   else
   {
      /* Turn it into a keyword now */
      pc.type = find_keyword_type(pc.str.c_str(), pc.str.size());
   }

   return(true);
}

/**
 * PAWN #define is different than C/C++.
 *   #define PATTERN REPLACEMENT_TEXT
 * The PATTERN may not contain a space or '[' or ']'.
 * A generic whitespace check should be good enough.
 * Do not change the pattern.
 */
static void parse_pawn_pattern(tok_ctx& ctx, chunk_t& pc, c_token_t tt)
{
   pc.str.clear();
   pc.type = tt;
   while (!unc_isspace(ctx.peek()))
   {
      pc.str.append(ctx.get());
   }
}

/**
 * Count the number of whitespace characters.
 *
 * @param pc   The structure to update, str is an input.
 * @return     Whether whitespace was parsed
 */
static bool parse_whitespace(tok_ctx& ctx, chunk_t& pc)
{
   int nl_count = 0;
   int ch       = -2;

   /* REVISIT: use a better whitespace detector? */
   while (ctx.more() && unc_isspace(ctx.peek()))
   {
      ch = ctx.get();   /* throw away the whitespace char */
      switch (ch)
      {
      case '\r':
         if (ctx.expect('\n'))
         {
            /* CRLF ending */
            cpd.le_counts[LE_CRLF]++;
         }
         else
         {
            /* CR ending */
            cpd.le_counts[LE_CR]++;
         }
         nl_count++;
         pc.orig_prev_sp = 0;
         break;

      case '\n':
         /* LF ending */
         cpd.le_counts[LE_LF]++;
         nl_count++;
         pc.orig_prev_sp = 0;
         break;

      case '\t':
         pc.orig_prev_sp += calc_next_tab_column(cpd.column, cpd.settings[UO_input_tab_size].n) - cpd.column;
         break;

      case ' ':
         pc.orig_prev_sp++;
         break;

      default:
         break;
      }
   }

   if (ch != -2)
   {
      pc.str.clear();
      pc.nl_count  = nl_count;
      pc.type      = nl_count ? CT_NEWLINE : CT_WHITESPACE;
      pc.after_tab = (ctx.c.last_ch == '\t');
      return(true);
   }
   return(false);
} // parse_whitespace

/**
 * Parses any number of tab or space chars followed by a newline.
 * Does not change pc.len if a newline isn't found.
 * This is not the same as parse_whitespace() because it only consumes until
 * a single newline is encountered.
 */
static bool parse_newline(tok_ctx& ctx)
{
   ctx.save();

   /* Eat whitespace */
   while ((ctx.peek() == ' ') || (ctx.peek() == '\t'))
   {
      ctx.get();
   }
   if ((ctx.peek() == '\r') || (ctx.peek() == '\n'))
   {
      if (!ctx.expect('\n'))
      {
         ctx.get();
         ctx.expect('\n');
      }
      return(true);
   }
   ctx.restore();
   return(false);
}