C++ ObScanner::get_is_req_fullfilled方法代码示例

void total_scan(const char *fname, PageArena<char> &allocer, MockClient &client, const char *version_range)
{
  ObScanner scanner;
  ObScanParam scan_param;
  read_scan_param(fname, SCAN_PARAM_SECTION, allocer, scan_param);
  scan_param.set_version_range(str2range(version_range));
  scan_param.set_is_read_consistency(false);

  int64_t total_fullfilled_num = 0;
  int64_t total_row_counter = 0;
  int64_t total_timeu = 0;
  while (true)
  {
    int64_t timeu = tbsys::CTimeUtil::getTime();
    int err = client.ups_scan(scan_param, scanner, timeout);
    timeu = tbsys::CTimeUtil::getTime() - timeu;
    if (OB_SUCCESS != err)
    {
      fprintf(stdout, "[%s] err=%d\n", __FUNCTION__, err);
      break;
    }
    else
    {
      int64_t row_counter = 0;
      while (OB_SUCCESS == scanner.next_cell())
      {
        ObCellInfo *ci = NULL;
        bool is_row_changed = false;
        scanner.get_cell(&ci, &is_row_changed);
        //fprintf(stdout, "%s\n", updateserver::print_cellinfo(ci, "CLI_SCAN"));
        if (is_row_changed)
        {
          row_counter++;
        }
      }
      bool is_fullfilled = false;
      int64_t fullfilled_num = 0;
      ObRowkey last_rk;
      scanner.get_last_row_key(last_rk);
      scanner.get_is_req_fullfilled(is_fullfilled, fullfilled_num);
      fprintf(stdout, "[SINGLE_SCAN] is_fullfilled=%s fullfilled_num=%ld row_counter=%ld timeu=%ld last_row_key=[%s]\n",
              STR_BOOL(is_fullfilled), fullfilled_num, row_counter, timeu, to_cstring(last_rk));
      total_fullfilled_num += fullfilled_num;
      total_row_counter += row_counter;
      total_timeu += timeu;
      if (is_fullfilled)
      {
        break;
      }
      else
      {
        const_cast<ObNewRange*>(scan_param.get_range())->start_key_ = last_rk;
        const_cast<ObNewRange*>(scan_param.get_range())->border_flag_.unset_min_value();
        const_cast<ObNewRange*>(scan_param.get_range())->border_flag_.unset_inclusive_start();
      }
    }
  }
  fprintf(stdout, "[TOTAL_SCAN] total_fullfilled_num=%ld total_row_counter=%ld total_timeu=%ld\n",
          total_fullfilled_num, total_row_counter, total_timeu);
}

 int get_ups_param(ObGetParam & param, const ObScanner & cs_result)
 {
   int err = OB_SUCCESS;
   ObReadParam &read_param = param;
   err =  get_ups_read_param(read_param,cs_result); 
   if (OB_SUCCESS == err)
   {
     bool is_fullfilled = false;
     int64_t fullfilled_cell_num  = 0;
     err = cs_result.get_is_req_fullfilled(is_fullfilled, fullfilled_cell_num);
     if (OB_SUCCESS == err && fullfilled_cell_num < param.get_cell_size())
     {
       err = param.rollback(param.get_cell_size() - fullfilled_cell_num);
       if ((err != OB_SUCCESS) || (param.get_cell_size() != fullfilled_cell_num))
       {
         TBSYS_LOG(WARN, "check param rollback failed:full_fill[%ld], param_size[%ld]",
             fullfilled_cell_num, param.get_cell_size());
       }
     }
   }
   return err;
 }

void print_scanner(ObScanner &scanner)
{
  int64_t row_counter = 0;
  while (OB_SUCCESS == scanner.next_cell())
  {
    ObCellInfo *ci = NULL;
    bool is_row_changed = false;
    scanner.get_cell(&ci, &is_row_changed);
    fprintf(stdout, "%s\n", common::print_cellinfo(ci, "CLI_SCAN"));
    if (is_row_changed)
    {
      row_counter++;
    }
  }
  bool is_fullfilled = false;
  int64_t fullfilled_num = 0;
  ObRowkey last_rk;
  scanner.get_last_row_key(last_rk);
  scanner.get_is_req_fullfilled(is_fullfilled, fullfilled_num);
  fprintf(stdout, "is_fullfilled=%s fullfilled_num=%ld row_counter=%ld last_row_key=[%s]\n",
          STR_BOOL(is_fullfilled), fullfilled_num, row_counter, to_cstring(last_rk));
}

int param_get(const char *fname, MockClient &client)
{
  int err = OB_SUCCESS;
  int fd = open(fname, O_RDONLY);
  if (-1 == fd)
  {
    fprintf(stdout, "open [%s] fail errno=%u\n", fname, errno);
    err = OB_ERROR;
  }
  else
  {
    struct stat st;
    fstat(fd, &st);
    char *buf = (char*)malloc(st.st_size);
    if (NULL == buf)
    {
      fprintf(stdout, "malloc buf fail size=%ld\n", st.st_size);
       err = OB_ERROR;
    }
    else
    {
      ssize_t __attribute__((unused)) ret = read(fd, buf, st.st_size);
      int64_t pos = 0;
      ObGetParam get_param;
      if (OB_SUCCESS != get_param.deserialize(buf, st.st_size, pos))
      {
        fprintf(stdout, "deserialize get_param fail\n");
        err = OB_ERROR;
      }
      else
      {
        get_param.set_is_read_consistency(false);
        ObScanner scanner;
        int err = client.ups_get(get_param, scanner, timeout);
        fprintf(stdout, "[%s] err=%d\n", __FUNCTION__, err);
        if (OB_SUCCESS == err)
        {
          int64_t row_counter = 0;
          while (OB_SUCCESS == scanner.next_cell())
          {
            ObCellInfo *ci = NULL;
            bool is_row_changed = false;
            scanner.get_cell(&ci, &is_row_changed);
            fprintf(stdout, "%s\n", common::print_cellinfo(ci, "CLI_GET"));
            if (is_row_changed)
            {
              row_counter++;
            }
          }
          bool is_fullfilled = false;
          int64_t fullfilled_num = 0;
          ObRowkey last_rk;
          scanner.get_last_row_key(last_rk);
          scanner.get_is_req_fullfilled(is_fullfilled, fullfilled_num);
          fprintf(stdout, "is_fullfilled=%s fullfilled_num=%ld row_counter=%ld last_row_key=[%s]\n",
                  STR_BOOL(is_fullfilled), fullfilled_num, row_counter, to_cstring(last_rk));
        }
      }
      free(buf);
    }
    close(fd);
  }
  return err;
}

TEST(TestObScanner, add_cell)
{
  ObScanner fem;
  fem.set_is_req_fullfilled(true, 1011);
  fem.set_data_version(170);

  char buf[BUFSIZ];
  int64_t pospos = 0;
  fem.serialize(buf, BUFSIZ, pospos);

  FILE *fd;
  fd = fopen("tmptest", "w+");
  fwrite(buf, 1, pospos, fd);
  fclose(fd);

  ObScanner os;
  ObScanner ost;
  /*** added by wushi ***/
  int64_t fullfilled_item = 0;
  int64_t tmp = 0;
  bool is_fullfilled = true;
  bool tmp_bool = false;
  /// initialize
  ASSERT_EQ(os.get_is_req_fullfilled(tmp_bool, fullfilled_item),0);
  EXPECT_FALSE(tmp_bool);
  ASSERT_EQ(fullfilled_item,0);
  /// set fullfilled
  is_fullfilled = true;
  fullfilled_item = 1000;
  os.set_is_req_fullfilled(is_fullfilled,fullfilled_item);
  ASSERT_EQ(os.get_is_req_fullfilled(tmp_bool, tmp),0);
  ASSERT_EQ(tmp_bool,is_fullfilled);
  ASSERT_EQ(tmp,fullfilled_item);

  /*** added by wushi ***/

  ObCellInfo oci;

  oci.table_name_.assign((char*)"table1", 6);
  oci.row_key_.assign((char*)"row1", 4);
  oci.column_name_.assign((char*)"column1", 7);
  oci.value_.set_int(0xff);

  ost.set_mem_size_limit(20);
  ASSERT_EQ(OB_SIZE_OVERFLOW, ost.add_cell(oci));
  os.set_mem_size_limit(1024 * 1024 * 2);
  ASSERT_EQ(OB_SUCCESS, os.add_cell(oci));

  oci.column_name_.assign((char*)"column2", 7);
  oci.value_.set_int(0xfe);
  ASSERT_EQ(OB_SUCCESS, os.add_cell(oci));

  ObScannerIterator iter = os.begin();
  ObCellInfo tci;
  ASSERT_EQ(OB_SUCCESS, iter.get_cell(tci));

  ASSERT_EQ(string("table1"), string(tci.table_name_.ptr(), tci.table_name_.length()));
  ASSERT_EQ(string("row1"), string(tci.row_key_.ptr(), tci.row_key_.length()));
  ASSERT_EQ(string("column1"), string(tci.column_name_.ptr(), tci.column_name_.length()));
  ASSERT_EQ(OB_SUCCESS, tci.value_.get_int(tmp));
  ASSERT_EQ(0xff, tmp);

  iter++;
  ASSERT_EQ(OB_SUCCESS, iter.get_cell(tci));

  ASSERT_EQ(string("table1"), string(tci.table_name_.ptr(), tci.table_name_.length()));
  ASSERT_EQ(string("row1"), string(tci.row_key_.ptr(), tci.row_key_.length()));
  ASSERT_EQ(string("column2"), string(tci.column_name_.ptr(), tci.column_name_.length()));
  ASSERT_EQ(OB_SUCCESS, tci.value_.get_int(tmp));
  ASSERT_EQ(0xfe, tmp);

  iter++;

  ASSERT_EQ(true, os.end() == iter);

  //ObCellInfo oci;
  char row_key_buffer[1024];

  oci.table_name_.assign((char*)"table1", 6);
  sprintf(row_key_buffer, "row1");
  oci.row_key_.assign(row_key_buffer, 4);
  oci.column_name_.assign((char*)"column1", 7);
  oci.value_.set_int(0xff);

  ASSERT_EQ(OB_SUCCESS, os.add_cell(oci));

  oci.table_name_.assign((char*)"table1", 6);
  sprintf(row_key_buffer, "row1");
  oci.row_key_.assign(row_key_buffer, 4);
  oci.column_name_.assign((char*)"column2", 7);
  oci.value_.set_int(0xee);

  os.set_mem_size_limit(20);
  ASSERT_EQ(OB_SIZE_OVERFLOW, os.add_cell(oci));
  os.set_mem_size_limit(1024 * 1024 * 2);
  ASSERT_EQ(OB_SUCCESS, os.add_cell(oci));

  oci.table_name_.assign((char*)"table1", 6);
  sprintf(row_key_buffer, "row2");
  oci.row_key_.assign(row_key_buffer, 4);
//.........这里部分代码省略.........

void scan_check_all(MutatorBuilder &mb, MockClient &client, const int64_t table_start_version, const bool using_id, const bool check)
{
  for (int64_t i = 0; i < mb.get_schema_num(); ++i)
  {
    ObScanParam scan_param;
    prepare_scan_param(scan_param, mb.get_schema(i), table_start_version, using_id);
    int ret = OB_SUCCESS;
    bool is_fullfilled = false;
    int64_t fullfilled_item_num = 0;
    RowChecker rc;
    int64_t row_counter = 0;
    ObScanner scanner;
    while (!is_fullfilled
          && OB_SUCCESS == ret)
    {
      int64_t timeu = tbsys::CTimeUtil::getTime();
      ret = client.ups_scan(scan_param, scanner, TIMEOUT_MS);
      TBSYS_LOG(INFO, "scan ret=%d timeu=%ld", ret, tbsys::CTimeUtil::getTime() - timeu);
      if (check
          && OB_SUCCESS == ret)
      {
        while (OB_SUCCESS == scanner.next_cell())
        {
          ObCellInfo *ci = NULL;
          bool is_row_changed = false;
          if (OB_SUCCESS == scanner.get_cell(&ci, &is_row_changed))
          {
            if (!using_id)
            {
              trans_name2id(*ci, mb.get_schema(i));
            }
            if (is_row_changed && 0 != rc.cell_num())
            {
              std::string row_key_str(rc.get_cur_rowkey().ptr(), 0, rc.get_cur_rowkey().length());
              //fprintf(stderr, "[%.*s] ", rc.get_cur_rowkey().length(), rc.get_cur_rowkey().ptr());

              bool get_row_bret = get_check_row(mb.get_schema(i), rc.get_cur_rowkey(), mb.get_cellinfo_builder(i), client, table_start_version, using_id);
              //fprintf(stderr, "[get_row check_ret=%d] ", bret);

              bool cell_info_bret = rc.check_row(mb.get_cellinfo_builder(i), mb.get_schema(i));
              //fprintf(stderr, "[cell_info check_ret=%d]\n", bret);
              TBSYS_LOG(INFO, "[%s] [get_row check_ret=%d] [cell_info check_ret=%d]", row_key_str.c_str(), get_row_bret, cell_info_bret);
            }
            if (is_row_changed
                && 0 != rc.rowkey_num()
                && rc.is_prefix_changed(ci->row_key_))
            {
              std::string row_key_str(rc.get_last_rowkey().ptr(), 0, rc.get_last_rowkey().length());
              //fprintf(stderr, "[%.*s] ", rc.get_last_rowkey().length(), rc.get_last_rowkey().ptr());

              bool bret = rc.check_rowkey(mb.get_rowkey_builder(i));
              //fprintf(stderr, "row_key check_ret=%d\n", bret);
              TBSYS_LOG(INFO, "[%s] [row_key check_ret=%d]", row_key_str.c_str(), bret);
            }
            rc.add_cell(ci);
            if (is_row_changed)
            {
              rc.add_rowkey(ci->row_key_);
              row_counter++;
            }
          }
        }
        if (0 != rc.cell_num())
        {
          std::string row_key_str(rc.get_cur_rowkey().ptr(), 0, rc.get_cur_rowkey().length());
          //fprintf(stderr, "[%.*s] ", rc.get_cur_rowkey().length(), rc.get_cur_rowkey().ptr());

          bool get_row_bret = get_check_row(mb.get_schema(i), rc.get_cur_rowkey(), mb.get_cellinfo_builder(i), client, table_start_version, using_id);
          //fprintf(stderr, "[get_row check_ret=%d] ", bret);

          bool cell_info_bret = rc.check_row(mb.get_cellinfo_builder(i), mb.get_schema(i));
          //fprintf(stderr, "[cell_info check_ret=%d]\n", bret);
          TBSYS_LOG(INFO, "[%s] [get_row check_ret=%d] [cell_info check_ret=%d]", row_key_str.c_str(), get_row_bret, cell_info_bret);
        }
      }
      scanner.get_is_req_fullfilled(is_fullfilled, fullfilled_item_num);
      ObRange *range = const_cast<ObRange*>(scan_param.get_range());
      scanner.get_last_row_key(range->start_key_);
      range->border_flag_.unset_min_value();
      range->border_flag_.unset_inclusive_start();
    }
    if (check
        && 0 != rc.rowkey_num())
    {
      std::string row_key_str(rc.get_last_rowkey().ptr(), 0, rc.get_last_rowkey().length());
      //fprintf(stderr, "[%.*s] ", rc.get_last_rowkey().length(), rc.get_last_rowkey().ptr());

      bool bret = rc.check_rowkey(mb.get_rowkey_builder(i));
      //fprintf(stderr, "row_key check_ret=%d\n", bret);
      TBSYS_LOG(INFO, "[%s] [row_key check_ret=%d]", row_key_str.c_str(), bret);
    }
    TBSYS_LOG(INFO, "table_id=%lu row_counter=%ld", mb.get_schema(i).get_table_id(), row_counter);
  }
}

int ObMergerRpcStub::scan(
    const int64_t timeout, const ObServer & server,
    const ObScanParam & scan_param, ObScanner & scanner) const
{
    int ret = OB_SUCCESS;
    ObDataBuffer data_buff;
    int64_t start_time = tbsys::CTimeUtil::getTime();
    ret = get_rpc_buffer(data_buff);
    // step 1. serialize ObScanParam to the data_buff
    if (OB_SUCCESS == ret)
    {
        ret = scan_param.serialize(data_buff.get_data(), data_buff.get_capacity(),
                                   data_buff.get_position());
        if (OB_SUCCESS != ret)
        {
            TBSYS_LOG(WARN, "serialize scan param failed:ret[%d]", ret);
        }
    }
    // step 2. send request for scan data
    if (OB_SUCCESS == ret)
    {
        ret = rpc_frame_->send_request(server, OB_SCAN_REQUEST, DEFAULT_VERSION,
                                       timeout, data_buff);
        if (ret != OB_SUCCESS)
        {
            TBSYS_LOG(WARN, "send scan request to server failed:ret[%d], timeout=%ld",
                      ret, timeout);
        }
    }
    // step 3. deserialize the response result
    int64_t pos = 0;
    if (OB_SUCCESS == ret)
    {
        ObResultCode result_code;
        ret = result_code.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
        if (OB_SUCCESS != ret)
        {
            TBSYS_LOG(WARN, "deserialize result failed:pos[%ld], ret[%d]", pos, ret);
        }
        else
        {
            ret = result_code.result_code_;
        }
    }

    // step 4. deserialize the scanner
    if (OB_SUCCESS == ret)
    {
        ret = scanner.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
        if (OB_SUCCESS != ret)
        {
            TBSYS_LOG(WARN, "deserialize scanner from buff failed:pos[%ld], ret[%d]", pos, ret);
        }
    }

    // write debug log
    const int32_t MAX_SERVER_ADDR_SIZE = 128;
    char server_addr[MAX_SERVER_ADDR_SIZE];
    server.to_string(server_addr, MAX_SERVER_ADDR_SIZE);
    bool is_fullfilled = false;
    int64_t fullfilled_item_num = 0;
    scanner.get_is_req_fullfilled(is_fullfilled,fullfilled_item_num);
    if (OB_SUCCESS == ret)
    {
        TBSYS_LOG(DEBUG, "scan data succ from server:addr[%s]", server_addr);
    }
    else
    {
        TBSYS_LOG(WARN, "scan data failed from server:addr[%s], "
                  "version_range=%s, ret[%d]",
                  server_addr, range2str(scan_param.get_version_range()), ret);
    }

    int64_t consume_time = tbsys::CTimeUtil::getTime() - start_time;
    if ((double)consume_time > (double)timeout * 0.8)
    {
        TBSYS_LOG(WARN, "slow ups scan, ups_addr=%s, timeout=%ld, consume=%ld",
                  server.to_cstring(), timeout, consume_time);
    }
    return ret;
}

int ObMergerSortedOperator::add_sharding_result(ObScanner & sharding_res, const ObNewRange &query_range,
  bool &is_finish)
{
  int err = OB_SUCCESS;
  if ((OB_SUCCESS == err) && (NULL == scan_param_))
  {
    TBSYS_LOG(WARN,"operator was not initialized yet [scan_param_:%p]", scan_param_);
    err = OB_INVALID_ARGUMENT;
  }
  if ((OB_SUCCESS == err) && (sharding_result_count_ >= MAX_SHARDING_RESULT_COUNT))
  {
    TBSYS_LOG(WARN,"array is full [MAX_SHARDING_RESULT_COUNT:%ld,sharding_result_count_:%ld]",
      MAX_SHARDING_RESULT_COUNT, sharding_result_count_);
    err = OB_ARRAY_OUT_OF_RANGE;
  }
  /// @todo (wushi [email protected]) check fullfill item number and size property
  bool is_fullfilled = false;
  int64_t fullfilled_item_num = 0;
  ObRowkey last_row_key;
  ObNewRange cs_tablet_range;
  if ((OB_SUCCESS == err) && (OB_SUCCESS != (err = sharding_res.get_is_req_fullfilled(is_fullfilled,fullfilled_item_num))))
  {
    TBSYS_LOG(WARN,"fail to get fullfilled info from sharding result [err:%d]", err);
  }
  if (OB_SUCCESS == err)
  {
    if (OB_SUCCESS == (err = sharding_res.get_last_row_key(last_row_key)))
    {
    }
    else if (OB_ENTRY_NOT_EXIST == err)
    {
      err = OB_SUCCESS;
    }
    else
    {
      TBSYS_LOG(WARN,"fail to get last rowkey from sharding result [err:%d]", err);
    }
  }

  if (OB_SUCCESS == err)
  {
    if (is_fullfilled)
    {
      if ((OB_SUCCESS == err) && (OB_SUCCESS != (err = sharding_res.get_range(cs_tablet_range))))
      {
        TBSYS_LOG(WARN,"fail to get tablet range from sharding result [err:%d]", err);
      }
      if (OB_SUCCESS == err)
      {
        if (ScanFlag::FORWARD == scan_param_->get_scan_direction())
        {
          last_row_key = cs_tablet_range.end_key_;
        }
        else
        {
          last_row_key = cs_tablet_range.start_key_;
        }
      }
    }
    if (OB_SUCCESS == err)
    {
      fullfilled_item_num = sharding_res.get_row_num();
      sharding_result_arr_[sharding_result_count_].init(sharding_res,query_range,*scan_param_, last_row_key, fullfilled_item_num);
      sharding_result_count_ ++;
    }
  }
  is_finish = false;
  if (OB_SUCCESS == err)
  {
    sort(is_finish, &sharding_res);
  }
  TBSYS_LOG(DEBUG, "add sharding result. is_finish=%d, err=%d", is_finish, err);
  return err;
}

 int get_next_param(
     const ObScanParam &org_scan_param, const ObScanner &prev_scan_result, 
     ObScanParam *scan_param, ObMemBuf &range_buffer)
 {
   int err = OB_SUCCESS;
   const ObReadParam &org_read_param = org_scan_param;
   ObReadParam *read_param = scan_param;
   ObNewRange tablet_range; 
   const ObNewRange *org_scan_range = NULL;
   ObNewRange cur_range;
   if(NULL != scan_param)
   {
     scan_param->reset();
     *read_param = org_read_param;
   }
   bool request_fullfilled = false;
   int64_t fullfilled_row_num = 0;
   err = prev_scan_result.get_is_req_fullfilled(request_fullfilled,fullfilled_row_num);
   if (OB_SUCCESS == err)
   {
     org_scan_range = org_scan_param.get_range();
     if (NULL == org_scan_range)
     {
       TBSYS_LOG(WARN, "unexpected error, can't get range from org scan param");
       err  = OB_INVALID_ARGUMENT;
     }
   }
   
   ObRowkey last_row_key;
   if (OB_SUCCESS == err)
   {
     if (request_fullfilled)
     {
       err = prev_scan_result.get_range(tablet_range);
       if (OB_SUCCESS == err)
       {
         if (true == is_finish_scan(org_scan_param, tablet_range))
         {
           err = OB_ITER_END;
         }
         else if (ScanFlag::FORWARD == org_scan_param.get_scan_direction()) 
         {
           last_row_key = tablet_range.end_key_;
         }
         else
         {
           last_row_key = tablet_range.start_key_;
         }
       }
     }
     else
     {
       err = prev_scan_result.get_last_row_key(last_row_key);
       if (OB_ENTRY_NOT_EXIST == err)
       {
         /// first time, remain unchanged
       }
       else if (OB_SUCCESS != err)
       {
         TBSYS_LOG(WARN, "unexpected error, scanner should contain at least one cell");
       }
     }
   }
 
   if (OB_SUCCESS == err && NULL != scan_param)
   {
     cur_range =*org_scan_range;
     // forward
     if (ScanFlag::FORWARD == org_scan_param.get_scan_direction())
     {
       cur_range.start_key_ = last_row_key;
       cur_range.border_flag_.unset_inclusive_start();
     }
     else
     {
       cur_range.end_key_ = last_row_key;
       if (request_fullfilled)
       {
         // tablet range start key
         cur_range.border_flag_.set_inclusive_end();
       }
       else
       {
         // the smallest rowkey
         cur_range.border_flag_.unset_inclusive_end();
       }
     }
   }
 
   // first time remain unchanged
   if (OB_ENTRY_NOT_EXIST == err)
   {
     cur_range =*org_scan_range;
     err = OB_SUCCESS;
   }
   
   if (OB_SUCCESS == err && NULL != scan_param)
   {
     err = allocate_range_buffer(cur_range, range_buffer);
   }
//.........这里部分代码省略.........