C++ ObScanner::deserialize方法代码示例

int ObRootServerRpcStub::scan(const ObServer & server, const int64_t timeout, 
                             const ObScanParam & param, ObScanner & result)
{
  ObDataBuffer data_buff;
  int ret = get_frame_buffer(data_buff);
  if (ret != OB_SUCCESS)
  {
    TBSYS_LOG(ERROR, "check get rpc buffer failed:ret[%d]", ret);
  }
  else
  {
    // step 1. serialize ObGetParam to the data_buff
    ret = param.serialize(data_buff.get_data(), data_buff.get_capacity(), 
        data_buff.get_position());
    if (OB_SUCCESS != ret)
    {
      TBSYS_LOG(ERROR, "serialize get param failed:ret[%d]", ret);
    }
  }

  // step 2. send request for get data 
  if (OB_SUCCESS == ret)
  {
    ret = rpc_frame_->send_request(server, OB_SCAN_REQUEST, DEFAULT_VERSION, 
        timeout, data_buff);
    if (ret != OB_SUCCESS)
    {
      TBSYS_LOG(WARN, "get data failed from server:ret[%d]", ret);
    }
  }
  
  // step 3. deserialize the response result
  int64_t pos = 0;
  if (OB_SUCCESS == ret)
  {
    ObResultCode result_code;
    ret = result_code.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
    if (OB_SUCCESS != ret)
    {
      TBSYS_LOG(ERROR, "deserialize result failed:pos[%ld], ret[%d]", pos, ret);
    }
    else if ((ret = result_code.result_code_) != OB_SUCCESS)
    {
      TBSYS_LOG(ERROR,"scan failed,ret[%d]",ret);
    }
  }
  
  // step 4. deserialize the scanner 
  if (OB_SUCCESS == ret)
  {
    result.clear();
    ret = result.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
    if (OB_SUCCESS != ret)
    {
      TBSYS_LOG(ERROR, "deserialize scanner from buff failed:pos[%ld], ret[%d]", pos, ret);
    }
  }
  return ret;
}

int print_scanner(const char * buffer, const int64_t len, int64_t & pos, FILE * file)
{
  int ret = OB_SUCCESS;
  if ((NULL == buffer) || (NULL == file))
  {
    TBSYS_LOG(ERROR, "check buffer failed:buff[%p], file[%p]", buffer, file);
    ret = OB_ERROR;
  }
  else
  {
    ObScanner result;
    ret = result.deserialize(buffer, len, pos);
    if (ret != OB_SUCCESS)
    {
      TBSYS_LOG(ERROR, "check deserialize scanner failed:pos[%ld], ret[%d]", pos, ret);
    }
    else
    {
      bool first_cell = true;
      ObCellInfo * cur_cell = NULL;
      ObCellInfo * pre_cell = NULL;
      while (result.next_cell() == OB_SUCCESS)
      {
        ret = result.get_cell(&cur_cell);
        if (OB_SUCCESS == ret)
        {
          pre_cell = cur_cell;
          if (first_cell)
          {
            first_cell = false;
            fprintf(file, "table_name:%.*s, rowkey:%.*s\n", cur_cell->table_name_.length(), cur_cell->table_name_.ptr(),
                cur_cell->row_key_.length(), cur_cell->row_key_.ptr());
          }
          /*
          fprintf(file, "table_name:%.*s, rowkey:%.*s, column_name:%.*s, ext:%ld, type:%d\n", 
              cur_cell->table_name_.length(), cur_cell->table_name_.ptr(),
              cur_cell->row_key_.length(), cur_cell->row_key_.ptr(), 
              cur_cell->column_name_.length(), cur_cell->column_name_.ptr(),
              cur_cell->value_.get_ext(),cur_cell->value_.get_type());
          //cur_cell->value_.dump();
          */
        }
        else
        {
          ret = OB_ERROR;
          TBSYS_LOG(ERROR, "get cell failed:ret[%d]", ret);
          break;
        }
      }

      if (OB_SUCCESS == ret)
      {
        fprintf(file, "table_name:%.*s, rowkey:%.*s\n", pre_cell->table_name_.length(), pre_cell->table_name_.ptr(),
            pre_cell->row_key_.length(), pre_cell->row_key_.ptr());
      }
    }
  }
  return ret;
}

void print_scanner(const char *fname)
{
  int fd = open(fname, O_RDONLY);
  if (-1 == fd)
  {
    fprintf(stderr, "open file [%s] fail errno=%u\n", fname, errno);
  }
  else
  {
    struct stat st;
    fstat(fd, &st);
    char *buffer = new char[st.st_size];
    if (NULL == buffer)
    {
      fprintf(stderr, "new buffer fail size=%ld\n", st.st_size);
    }
    else
    {
      int64_t read_ret = read(fd, buffer, st.st_size);
      if (st.st_size != read_ret)
      {
        fprintf(stderr, "read file fail ret=%ld size=%ld errno=%u\n", read_ret, st.st_size, errno);
      }
      else
      {
        int64_t pos = 0;
        ObScanner scanner;
        int ret = scanner.deserialize(buffer, st.st_size, pos);
        if (OB_SUCCESS != ret)
        {
          fprintf(stderr, "deserialize fail ret=%d\n", ret);
        }
        else
        {
          print_scanner(scanner);
        }
      }
      delete[] buffer;
    }
    close(fd);
  }
}

int rpc_cs_get(ObClientManager& cp,
               const ObServer& cs,
               const ObGetParam& get_param,
               ObScanner& scanner)
{
  int ret = OB_SUCCESS;
  int64_t start = 0, end = 0;
  const int32_t BUFFER_SIZE = 2*1024*1024;
  char* param_buffer = new char[BUFFER_SIZE];
  ObDataBuffer ob_inout_buffer;
  ObResultCode rc;
  int64_t return_start_pos = 0;

  if (NULL == param_buffer)
  {
    goto exit;
  }

  ob_inout_buffer.set_data(param_buffer, BUFFER_SIZE);
  ret = get_param.serialize(ob_inout_buffer.get_data(),
                            ob_inout_buffer.get_capacity(), ob_inout_buffer.get_position());
  if (OB_SUCCESS != ret)
  {
    fprintf(stderr,"serialize get_param into buffer failed\n");
    goto exit;
  }

  // send request;
  start = tbsys::CTimeUtil::getTime();
  ret = cp.send_request(cs, OB_GET_REQUEST, 1, 2000*2000, ob_inout_buffer);
  end = tbsys::CTimeUtil::getTime();
  fprintf(stderr,"time consume:%ld\n", end - start);
  if (OB_SUCCESS != ret)
  {
    fprintf(stderr,"rpc failed\n");
    goto exit;
  }

  ret = rc.deserialize(ob_inout_buffer.get_data(),
                       ob_inout_buffer.get_position(), return_start_pos);

  if (OB_SUCCESS != ret)
  {
    fprintf(stderr,"deserialize obscanner failed\n");
    goto exit;
  }

  fprintf(stderr,"return rc code:%d, msg:%s\n", rc.result_code_, rc.message_.ptr());

  if (OB_SUCCESS != rc.result_code_)
  {
    goto exit;
  }

  fprintf(stderr,"return_start_pos:%ld, %ld\n", return_start_pos, ob_inout_buffer.get_position());

  // deserialize output
  ret = scanner.deserialize(ob_inout_buffer.get_data(),
                            ob_inout_buffer.get_position(), return_start_pos);
  if (OB_SUCCESS != ret)
  {
    fprintf(stderr,"deserialize obscanner failed\n");
    goto exit;
  }
  fprintf(stderr,"return_start_pos:%ld, %ld\n", return_start_pos, ob_inout_buffer.get_position());

exit:
  if (param_buffer) delete []param_buffer;
  return ret;
}

    int ObServerRpc::get(const ObServer& remote_server,
                         const ObGetParam& get_param, 
                         ObScanner& scanner,
                         const int64_t timeout)
    {
      int ret     = OB_SUCCESS;
      int64_t pos = 0;
      ObResultCode result_code;
      ObDataBuffer data_buff;

      if (remote_server.get_ipv4() == 0 || remote_server.get_port() == 0
          || timeout <= 0)
      {
        TBSYS_LOG(WARN, "invalid param, ip=%d, port=%d, timeout=%ld",
                  remote_server.get_ipv4(), remote_server.get_port(),
                  timeout);
        ret = OB_ERROR;
      }
      else if (get_param.get_cell_size() <= 0 || get_param.get_row_size() <= 0)
      {
        TBSYS_LOG(WARN, "invalid param, cell_size=%ld, row_size=%ld", 
                  get_param.get_cell_size(), get_param.get_row_size());
        ret = OB_ERROR;
      }
      else if (NULL == rpc_frame_)
      {
        TBSYS_LOG(WARN, "server rpc doesn't init.");
        ret = OB_ERROR;
      }

      if (OB_SUCCESS == ret)
      {
        ret = get_frame_buffer(data_buff);
        if (OB_SUCCESS != ret)
        {
          TBSYS_LOG(WARN, "get frame buffer failed, ret=%d.", ret);
        }
      }
      
      // step 1. serialize get param to data_buff
      if (OB_SUCCESS == ret)
      {
        ret = get_param.serialize(data_buff.get_data(), 
                                  data_buff.get_capacity(), 
                                  data_buff.get_position());
        if (OB_SUCCESS != ret)
        {
          TBSYS_LOG(WARN, "serialize get_param failed, ret=%d.", ret);
        }
      }
    
      // step 2. send request for get
      if (OB_SUCCESS == ret)
      {
        ret = rpc_frame_->send_request(remote_server, 
                                       OB_GET_REQUEST, DEFAULT_VERSION, 
                                       timeout, data_buff);
        if (OB_SUCCESS != ret)
        {
          TBSYS_LOG(WARN, "send request to remote server for get failed, "
                          "ret=%d.", ret);
        }
      }
    
      // step 3. deserialize the response code
      if (OB_SUCCESS == ret)
      {
        ret = result_code.deserialize(data_buff.get_data(), 
                                      data_buff.get_position(), pos);
        if (OB_SUCCESS != ret)
        {
          TBSYS_LOG(WARN, "deserialize result_code failed,pos=%ld, ret=%d.", 
                    pos, ret);
        }
        else
        {
          ret = result_code.result_code_;
          if (OB_SUCCESS != ret && OB_DATA_NOT_SERVE != ret)
          {
            TBSYS_LOG(WARN, "get response from remote server failed, "
                            "timeout=%ld, ret=%d.", timeout, ret);
          }
        }
      }

      // step 4. deserialize the scanner
      if (OB_SUCCESS == ret)
      {
        ret = scanner.deserialize(data_buff.get_data(), 
                                  data_buff.get_position(), pos);
        if (OB_SUCCESS != ret)
        {
          TBSYS_LOG(WARN, "deserialize scanner from buff failed, "
                          "pos=%ld, ret=%d.", pos, ret);
        }
      }
    
      return ret;
    }

//.........这里部分代码省略.........
        ci.table_name_.length(), ci.table_name_.ptr(),
        ci.row_key_.length(), ci.row_key_.ptr(),
        ci.column_name_.length(), ci.column_name_.ptr());
    ASSERT_EQ(OB_SUCCESS, iter.get_cell(ci));
    fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s]\n",
        ci.table_name_.length(), ci.table_name_.ptr(),
        ci.row_key_.length(), ci.row_key_.ptr(),
        ci.column_name_.length(), ci.column_name_.ptr());
  }
  fprintf(stdout, "==============================\n");
  while (OB_SUCCESS == os.next_cell())
  {
    ObCellInfo *ci = NULL;
    bool is_row_changed;
    ASSERT_EQ(OB_SUCCESS, os.get_cell(&ci, &is_row_changed));
    fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s] "
            "table_id=[%lu] column_id=[%lu] row_changed=[%d]\n",
            ci->table_name_.length(), ci->table_name_.ptr(),
            ci->row_key_.length(), ci->row_key_.ptr(),
            ci->column_name_.length(),ci->column_name_.ptr(),
            ci->table_id_, ci->column_id_, is_row_changed);
    ASSERT_EQ(OB_SUCCESS, os.get_cell(&ci, &is_row_changed));
    fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s] "
            "table_id=[%lu] column_id=[%lu] row_changed=[%d]\n",
            ci->table_name_.length(), ci->table_name_.ptr(),
            ci->row_key_.length(), ci->row_key_.ptr(),
            ci->column_name_.length(),ci->column_name_.ptr(),
            ci->table_id_, ci->column_id_, is_row_changed);
  }
  os.reset_iter();
  fprintf(stdout, "==============================\n");

  pos = 0;
  ASSERT_EQ(OB_SUCCESS, os.deserialize(buffer, os.get_serialize_size(), pos));
  ASSERT_EQ(pos, os.get_serialize_size());

  for (iter = os.begin(); iter != os.end(); iter++)
  {
    ObCellInfo ci;
    ASSERT_EQ(OB_SUCCESS, iter.get_cell(ci));
    fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s]\n",
        ci.table_name_.length(), ci.table_name_.ptr(),
        ci.row_key_.length(), ci.row_key_.ptr(),
        ci.column_name_.length(), ci.column_name_.ptr());
    ASSERT_EQ(OB_SUCCESS, iter.get_cell(ci));
    fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s]\n",
        ci.table_name_.length(), ci.table_name_.ptr(),
        ci.row_key_.length(), ci.row_key_.ptr(),
        ci.column_name_.length(), ci.column_name_.ptr());
  }
  fprintf(stdout, "==============================\n");
  while (OB_SUCCESS == os.next_cell())
  {
    ObCellInfo *ci = NULL;
    bool is_row_changed;
    ASSERT_EQ(OB_SUCCESS, os.get_cell(&ci, &is_row_changed));
    fprintf(stdout, "table_name=[%.*s] row_key=[%.*s] column_name=[%.*s] "
            "table_id=[%lu] column_id=[%lu] row_changed=[%d]\n",
            ci->table_name_.length(), ci->table_name_.ptr(),
            ci->row_key_.length(), ci->row_key_.ptr(),
            ci->column_name_.length(),ci->column_name_.ptr(),
            ci->table_id_, ci->column_id_, is_row_changed);
  }
  os.reset_iter();

  /// set fullfilled

TEST(TestObScanner, get_row)
{
  ObScanner scanner;
  const int table_num = 10;
  const int row_num = 20;
  const int column_num = 10;
  buffer table_name[table_num];
  buffer row_key[row_num];
  buffer column_name[column_num];
  for (int i = 0; i < table_num; i++)
  {
    table_name[i].assigne("T").appende(i);
  }
  for (int i = 0; i < row_num; i++)
  {
    row_key[i].assigne("R").appende(i);
  }
  for (int i = 0; i < column_num; i++)
  {
    column_name[i].assigne("C").appende(i);
  }

  for (int i = 0; i < table_num; i++)
  {
    for (int j = 0; j < row_num; j++)
    {
      for (int k = 0; k < column_num; k++)
      {
        ObCellInfo cell;
        cell.table_name_ = table_name[i].get_obstring();
        cell.row_key_ = row_key[j].get_obstring();
        cell.column_name_ = column_name[k].get_obstring();
        cell.value_.set_int(1);
        scanner.add_cell(cell);
      }
    }
  }

  ObScanner::RowIterator it = scanner.row_begin();
  ASSERT_EQ(it ==  scanner.row_begin(), true);

  for (int i = 0; i < table_num; i++)
  {
    for (int j = 0; j < row_num; j++)
    {
      ObCellInfo *row;
      int64_t num = 0;
      ASSERT_EQ(OB_SUCCESS, scanner.next_row());
      ASSERT_EQ(OB_SUCCESS, scanner.get_row(&row, &num));
      ASSERT_EQ(column_num, num);
    }
  }
  ASSERT_EQ(OB_ITER_END, scanner.next_row());

  buffer buf(1 << 21);
  scanner.serialize(buf.ptre(), buf.capacity(), buf.length());
  printf("%ld\n", buf.length());
  ObScanner dscanner;
  int64_t pos = 0;
  dscanner.deserialize(buf.ptre(), buf.length(), pos);

  for (int i = 0; i < table_num; i++)
  {
    for (int j = 0; j < row_num; j++)
    {
      ObCellInfo *row;
      int64_t num = 0;
      ASSERT_EQ(OB_SUCCESS, dscanner.next_row());
      ASSERT_EQ(OB_SUCCESS, dscanner.get_row(&row, &num));
      ASSERT_EQ(column_num, num);
    }
  }
  ASSERT_EQ(OB_ITER_END, dscanner.next_row());

  scanner.clear();
  scanner.set_mem_size_limit(256);
  int j = 0, k = 0;

  int err = OB_SUCCESS;
  for (; j < row_num; j++)
  {
    for (k = 0; k < column_num; k++)
    {
      ObCellInfo cell;
      cell.table_name_ = table_name[0].get_obstring();
      cell.row_key_ = row_key[j].get_obstring();
      cell.column_name_ = column_name[k].get_obstring();
      cell.value_.set_int(1);
      if (OB_SUCCESS != (err = scanner.add_cell(cell)))
      {
        break;
      }
    }
    if (OB_SUCCESS != err)
    {
      break;
    }
  }
  if (OB_SUCCESS != err)
  {
//.........这里部分代码省略.........

int ObClientServerStub::cs_get(const ObGetParam& get_param, ObScanner& scanner)
{
  int ret = OB_SUCCESS;
  const int64_t timeout = 1000000;  // send_request timeout millionseconds
  ObDataBuffer data_buff;
  ret = get_frame_buffer(data_buff);

  // step 1. serialize timestamp to data_buff
  if (OB_SUCCESS == ret)
  {
    ret = get_param.serialize(data_buff.get_data(), 
        data_buff.get_capacity(), data_buff.get_position());
    if (OB_SUCCESS != ret)
    {
      TBSYS_LOG(ERROR, "serialize get_param failed:ret[%d].", ret);
    }
  }

  // step 2. send request for fetch new schema
  if (OB_SUCCESS == ret)
  {
    uint32_t pos = random() % merge_server_list_.size();
    ObServer &cur_server = merge_server_list_.at(pos);

    ret = rpc_frame_->send_request(cur_server, 
        OB_GET_REQUEST, DEFAULT_VERSION, timeout, data_buff);
    if (ret != OB_SUCCESS)
    {
      TBSYS_LOG(ERROR, "send request to remote server for get failed"
          " ret[%d].",  ret);
    }
  }


  // step 3. deserialize the response code
  int64_t pos = 0;
  if (OB_SUCCESS == ret)
  {
    ObResultCode result_code;
    ret = result_code.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
    if (OB_SUCCESS != ret)
    {
      TBSYS_LOG(ERROR, "deserialize result_code failed:pos[%ld], ret[%d].", pos, ret);
    }
    else
    {
      ret = result_code.result_code_;
    }
  }
  // step 4. deserialize the table schema
  if (OB_SUCCESS == ret)
  {
    ret = scanner.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
    if (OB_SUCCESS != ret)
    {
      TBSYS_LOG(ERROR, "deserialize scanner from buff failed"
          "pos[%ld], ret[%d].", pos, ret);
    }
  }

  return ret;
}

int ObMergerRpcStub::scan(
    const int64_t timeout, const ObServer & server,
    const ObScanParam & scan_param, ObScanner & scanner) const
{
    int ret = OB_SUCCESS;
    ObDataBuffer data_buff;
    int64_t start_time = tbsys::CTimeUtil::getTime();
    ret = get_rpc_buffer(data_buff);
    // step 1. serialize ObScanParam to the data_buff
    if (OB_SUCCESS == ret)
    {
        ret = scan_param.serialize(data_buff.get_data(), data_buff.get_capacity(),
                                   data_buff.get_position());
        if (OB_SUCCESS != ret)
        {
            TBSYS_LOG(WARN, "serialize scan param failed:ret[%d]", ret);
        }
    }
    // step 2. send request for scan data
    if (OB_SUCCESS == ret)
    {
        ret = rpc_frame_->send_request(server, OB_SCAN_REQUEST, DEFAULT_VERSION,
                                       timeout, data_buff);
        if (ret != OB_SUCCESS)
        {
            TBSYS_LOG(WARN, "send scan request to server failed:ret[%d], timeout=%ld",
                      ret, timeout);
        }
    }
    // step 3. deserialize the response result
    int64_t pos = 0;
    if (OB_SUCCESS == ret)
    {
        ObResultCode result_code;
        ret = result_code.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
        if (OB_SUCCESS != ret)
        {
            TBSYS_LOG(WARN, "deserialize result failed:pos[%ld], ret[%d]", pos, ret);
        }
        else
        {
            ret = result_code.result_code_;
        }
    }

    // step 4. deserialize the scanner
    if (OB_SUCCESS == ret)
    {
        ret = scanner.deserialize(data_buff.get_data(), data_buff.get_position(), pos);
        if (OB_SUCCESS != ret)
        {
            TBSYS_LOG(WARN, "deserialize scanner from buff failed:pos[%ld], ret[%d]", pos, ret);
        }
    }

    // write debug log
    const int32_t MAX_SERVER_ADDR_SIZE = 128;
    char server_addr[MAX_SERVER_ADDR_SIZE];
    server.to_string(server_addr, MAX_SERVER_ADDR_SIZE);
    bool is_fullfilled = false;
    int64_t fullfilled_item_num = 0;
    scanner.get_is_req_fullfilled(is_fullfilled,fullfilled_item_num);
    if (OB_SUCCESS == ret)
    {
        TBSYS_LOG(DEBUG, "scan data succ from server:addr[%s]", server_addr);
    }
    else
    {
        TBSYS_LOG(WARN, "scan data failed from server:addr[%s], "
                  "version_range=%s, ret[%d]",
                  server_addr, range2str(scan_param.get_version_range()), ret);
    }

    int64_t consume_time = tbsys::CTimeUtil::getTime() - start_time;
    if ((double)consume_time > (double)timeout * 0.8)
    {
        TBSYS_LOG(WARN, "slow ups scan, ups_addr=%s, timeout=%ld, consume=%ld",
                  server.to_cstring(), timeout, consume_time);
    }
    return ret;
}