C++ ObScanParam::add_column方法代码示例

int ObMetaTable3::MyIterator::add_scan_columns(ObScanParam &scan_param)
{
  int ret = OB_SUCCESS;
  if (OB_SUCCESS != (ret = get_table_schema()))
  {
    TBSYS_LOG(WARN, "failed to get table schema, err=%d", ret);
  }
  else
  {
    ObMetaTableColumnSchema col_schema(table_schema_);
    for (int64_t i = 0; i < col_schema.get_columns_num(); ++i)
    {
      uint64_t cid = col_schema.get_cid_by_idx(i);
      if (OB_INVALID_ID == cid)
      {
        TBSYS_LOG(ERROR, "failed to get column id, i=%ld", i);
      }
      else if (OB_SUCCESS != (ret = scan_param.add_column(cid)))
      {
        TBSYS_LOG(WARN, "failed to add column, err=%d", ret);
        break;
      }
    }
  }
  return ret;
}

int MutatorBuilder::build_total_scan_param(ObScanParam &scan_param, PageArena<char> &allocer,
                                          const int64_t table_start_version,
                                          const bool using_id, const int64_t table_pos)
{
  int ret = OB_SUCCESS;
  const ObSchema &schema = schema_mgr_.begin()[table_pos];

  int64_t cur_rowkey_info = get_cur_rowkey_info_(schema, using_id, rb_array_[table_pos]->get_prefix_start());
  std::pair<ObString,ObString> key_range = rb_array_[table_pos]->get_rowkey4total_scan(cur_rowkey_info, allocer);
  ObRange range;
  range.start_key_ = key_range.first;
  range.end_key_ = key_range.second;

  const ObColumnSchema *iter = NULL;
  for (iter = schema.column_begin(); iter != schema.column_end(); iter++)
  {
    if (using_id)
    {
      scan_param.add_column(iter->get_id());
    }
    else
    {
      ObString column_name;
      column_name.assign_ptr(const_cast<char*>(iter->get_name()), static_cast<int32_t>(strlen(iter->get_name())));
      scan_param.add_column(column_name);
    }
  }

  if (using_id)
  {
    scan_param.set(schema.get_table_id(), ObString(), range);
  }
  else
  {
    ObString table_name;
    table_name.assign_ptr(const_cast<char*>(schema.get_table_name()), static_cast<int32_t>(strlen(schema.get_table_name())));
    scan_param.set(OB_INVALID_ID, table_name, range);
  }

  ObVersionRange version_range;
  version_range.start_version_ = table_start_version;
  version_range.border_flag_.set_max_value();
  version_range.border_flag_.set_inclusive_start();
  scan_param.set_version_range(version_range);
  return ret;
}

void prepare_scan_param(ObScanParam &scan_param, const ObSchema &schema,
                        const int64_t table_start_version, const bool using_id)
{
  ObRange range;
  range.border_flag_.set_min_value();
  range.border_flag_.set_max_value();

  const ObColumnSchema *iter = NULL;
  for (iter = schema.column_begin(); iter != schema.column_end(); iter++)
  {
    if (using_id)
    {
      scan_param.add_column(iter->get_id());
    }
    else
    {
      ObString column_name;
      column_name.assign_ptr(const_cast<char*>(iter->get_name()), strlen(iter->get_name()));
      scan_param.add_column(column_name);
    }
  }

  if (using_id)
  {
    scan_param.set(schema.get_table_id(), ObString(), range);
  }
  else
  {
    ObString table_name;
    table_name.assign(const_cast<char*>(schema.get_table_name()), strlen(schema.get_table_name()));
    scan_param.set(OB_INVALID_ID, table_name, range);
  }

  ObVersionRange version_range;
  version_range.start_version_ = table_start_version;
  version_range.border_flag_.set_max_value();
  version_range.border_flag_.set_inclusive_start();
  scan_param.set_version_range(version_range);
}

int TaskFactory::add_all_columns(const uint64_t table_id, ObScanParam & param)
{
  int ret = OB_SUCCESS;
  int32_t count = 0;
  const ObColumnSchemaV2 * column_schema = schema_->get_table_schema(table_id, count);
  if ((NULL == column_schema) || (0 == count))
  {
    ret = OB_ERROR;
    TBSYS_LOG(ERROR, "check column schema or schema count failed:column[%p], count[%d]",
        column_schema, count);
  }
  else
  {
    ObString column;
    const char * column_name = NULL;
    for (int32_t i = 0; i < count; ++i)
    {
      column_name = column_schema->get_name();
      if (column_name != NULL)
      {
        TBSYS_LOG(DEBUG, "TableId:%ld, COLUMN=%s", table_id, column_name);

        column.assign(const_cast<char *>(column_name), (int32_t)strlen(column_name));
        ret = param.add_column(column);
        if (ret != OB_SUCCESS)
        {
          TBSYS_LOG(ERROR, "add column failed:table[%lu], column[%s], ret[%d]",
              table_id, column_name, ret);
          ret = OB_ERROR;
          break;
        }
      }
      else
      {
        ret = OB_ERROR;
        TBSYS_LOG(ERROR, "check column name failed:table[%lu], name[%s]", table_id, column_name);
        break;
      }
      ++column_schema;
    }
  }
  return ret;
}

int scan_test_case_common(
  BaseClient& client,
  const ObServer& server,
  const int64_t  table_id,
  const char* start_key_ptr,
  const char* end_key_ptr,
  const int64_t input_key_size,
  const bool is_hex
  )
{
  ObScanParam input;


  int64_t key_size = input_key_size;
  if (is_hex)
  {
    key_size = input_key_size / 2;
  }

  char start_key[input_key_size];
  char end_key[input_key_size];
  char range_str[OB_RANGE_STR_BUFSIZ];


  memset(start_key, 0, input_key_size);
  memset(end_key, 0, input_key_size);
  if (is_hex)
  {
    str_to_hex(start_key_ptr, static_cast<int32_t>(input_key_size), start_key, static_cast<int32_t>(key_size));
    str_to_hex(end_key_ptr, static_cast<int32_t>(input_key_size), end_key, static_cast<int32_t>(key_size));
  }
  else
  {
    memcpy(start_key, start_key_ptr, key_size);
    memcpy(end_key, end_key_ptr, key_size);
  }


  ObString ob_table_name(0, 0, NULL);

  ObRange range;
  range.table_id_= 0;
  range.start_key_.assign_ptr(start_key, static_cast<int32_t>(key_size));
  range.end_key_.assign_ptr(end_key, static_cast<int32_t>(key_size));
  range.border_flag_.set_inclusive_start();
  range.border_flag_.set_inclusive_end();

  range.to_string(range_str, OB_RANGE_STR_BUFSIZ);
  range.hex_dump();
  fprintf(stderr, "input scan range = %s\n", range_str);


  input.set(table_id, ob_table_name, range);
  input.set_scan_size(100000);
  input.add_column((uint64_t)0ull);


  ObScanner scanner;
  int64_t start = tbsys::CTimeUtil::getTime();
  int ret = rpc_cs_scan(client.client_, server, input, scanner);
  int64_t end = tbsys::CTimeUtil::getTime();
  fprintf(stderr,"rpc_cs_scan time consume:%ld\n", end - start);
  if (OB_SUCCESS != ret)
  {
    fprintf(stderr,"ret:%d\n", ret);
  }
  else
  {
    dump_scanner(scanner);
  }
  
  return ret;
}

int scan_test_case(
    BaseClient& client,
    const ObServer& server,
    const uint64_t table_id,
    const int64_t user_id,
  const int8_t item_type,
  const int32_t column_size
  )
{
  ObScanParam input;


  const int32_t key_size = 17;
  char start_key[key_size] ;
  char end_key[key_size] ;

  int64_t pos = 0;
  
  encode_i64(start_key, key_size, pos, user_id);
  if (item_type <= 1)
    encode_i8(start_key, key_size, pos, item_type);
  else
    encode_i8(start_key, key_size, pos, 0x0);
  memset(start_key + pos, 0x0, key_size - pos);

  pos = 0;
  encode_i64(end_key, key_size, pos, user_id);
  if ( item_type <= 1)
    encode_i8(end_key, key_size, pos, item_type);
  else
    encode_i8(end_key, key_size, pos, static_cast<int8_t>(0xFF));

  memset(end_key + pos, 0xFF, key_size - pos);

  hex_dump(start_key, key_size);
  hex_dump(end_key, key_size);


  ObString ob_table_name;
  ObRange range;
  range.table_id_= table_id;
  range.start_key_.assign_ptr(start_key, key_size);
  range.end_key_.assign_ptr(end_key, key_size);
  range.border_flag_.set_inclusive_start();
  range.border_flag_.set_inclusive_end();

  range.hex_dump();

  input.set(table_id, ob_table_name, range);
  input.set_scan_size(100000);

  for (int i = 0; i < column_size; ++i)
  {
    input.add_column(i + 2);
  }

  ObScanner scanner;
  int64_t start = tbsys::CTimeUtil::getTime();
  int ret = rpc_cs_scan(client.client_, server, input, scanner);
  int64_t end = tbsys::CTimeUtil::getTime();
  fprintf(stderr,"rpc_cs_scan time consume:%ld\n", end - start);
  if (OB_SUCCESS != ret)
  {
    fprintf(stderr,"ret:%d\n", ret);
  }
  else
  {
    dump_scanner(scanner);
  }
  
  return ret;
}

void read_scan_param(const char *fname, const char *section, PageArena<char> &allocer, ObScanParam &scan_param)
{
  TBSYS_CONFIG.load(fname);
  const char *str = NULL;
  int32_t len = 0;

  str = TBSYS_CONFIG.getString(section, SCAN_FROZEN_ONLY);
  if (0 == strcmp("YES", str))
  {
    //scan_param.set_is_read_frozen_only(true);
  }

  str = TBSYS_CONFIG.getString(section, SCAN_REVERSE);
  if (NULL == str
      || 0 != strcmp("YES", str))
  {
    scan_param.set_scan_direction(ObScanParam::FORWARD);
  }
  else
  {
    scan_param.set_scan_direction(ObScanParam::BACKWARD);
  }

  ObString table_name;
  sprintf(buffer, SCAN_TABLE_NAME);
  str = getAndCopySting_(allocer, section, buffer, len);
  table_name.assign_ptr(const_cast<char*>(str), len);

  uint64_t table_id = OB_INVALID_ID;
  if (NULL == str)
  {
    table_id = TBSYS_CONFIG.getInt(section, SCAN_TABLE_ID);
  }
  else
  {
    table_id = OB_INVALID_ID;
  }

  int32_t columns_num = TBSYS_CONFIG.getInt(section, SCAN_COLUMNS_NUM);
  for (int32_t i = 0; i < columns_num; i++)
  {
    ObString column_name;
    sprintf(buffer, SCAN_COLUMN_NAME_KEY_FORMAT, i);
    str = getAndCopySting_(allocer, section, buffer, len);
    column_name.assign_ptr(const_cast<char*>(str), len);
    if (NULL != str)
    {
      scan_param.add_column(column_name);
    }
    else
    {
      sprintf(buffer, SCAN_COLUMN_ID_KEY_FORMAT, i);
      uint64_t column_id = TBSYS_CONFIG.getInt(section, buffer);
      scan_param.add_column(column_id);
    }
  }

  ObRange range;
  range.table_id_ = table_id;
  
  str = getAndCopySting_(allocer, section, SCAN_START_KEY, len);
  range.start_key_.assign_ptr(const_cast<char*>(str), len);
  if (0 == strcmp("MIN_KEY", str))
  {
    range.border_flag_.set_min_value();
  }
  if (0 == strcmp("MAX_KEY", str))
  {
    range.border_flag_.set_max_value();
  }
  
  str = getAndCopySting_(allocer, section, SCAN_END_KEY, len);
  range.end_key_.assign_ptr(const_cast<char*>(str), len);
  if (0 == strcmp("MIN_KEY", str))
  {
    range.border_flag_.set_min_value();
  }
  if (0 == strcmp("MAX_KEY", str))
  {
    range.border_flag_.set_max_value();
  }

  str = getAndCopySting_(allocer, section, SCAN_INCLUSIVE_START, len);
  if (0 == strcmp("YES", str))
  {
    range.border_flag_.set_inclusive_start();
  }
  str = getAndCopySting_(allocer, section, SCAN_INCLUSIVE_END, len);
  if (0 == strcmp("YES", str))
  {
    range.border_flag_.set_inclusive_end();
  }

  scan_param.set(table_id, table_name, range);
}

void init_decode_param(ObStringBuf &buf,  ObScanParam &org_scan_param)
{
  UNUSED(buf);
  const char *c_ptr;
  ObString str;
  ObNewRange q_range;
  q_range.set_whole_range();
  q_range.border_flag_.set_inclusive_start();
  q_range.border_flag_.set_inclusive_end();
  c_ptr = "collect_info";
  str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
  org_scan_param.reset();
  EXPECT_EQ(org_scan_param.set(OB_INVALID_ID, str,q_range), OB_SUCCESS);

  /// basic columns
  bool is_return = false;
  c_ptr = "item_collect_count";
  str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
  EXPECT_EQ(org_scan_param.add_column(str,is_return), OB_SUCCESS );

  is_return = false;
  c_ptr = "item_category";
  str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
  EXPECT_EQ(org_scan_param.add_column(str,is_return), OB_SUCCESS );

  is_return = true;
  c_ptr = "item_price";
  str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
  EXPECT_EQ(org_scan_param.add_column(str,is_return), OB_SUCCESS );

  /// composite columns
  c_ptr = "`item_collect_count`*`item_price`";
  str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
  EXPECT_EQ(org_scan_param.add_column(str,str, is_return), OB_SUCCESS );

  /// where condition
  c_ptr = "`item_price` > 10";
  str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
  EXPECT_EQ(org_scan_param.add_where_cond(str), OB_SUCCESS );

  /// groupby::group by columns
  is_return = false;
  c_ptr = "item_category";
  str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
  EXPECT_EQ(org_scan_param.get_group_by_param().add_groupby_column(str,is_return), OB_SUCCESS);

  /// aggregate columns
  is_return = true;
  c_ptr = "item_price";
  str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
  EXPECT_EQ(org_scan_param.get_group_by_param().add_aggregate_column(str,str,  SUM, is_return), OB_SUCCESS);

  /// aggregate columns
  is_return = false;
  c_ptr = "item_collect_count";
  str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
  EXPECT_EQ(org_scan_param.get_group_by_param().add_aggregate_column(str,str,  SUM, is_return), OB_SUCCESS);

  /// composite columns
  c_ptr = "`item_collect_count`*`item_price`";
  str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
  EXPECT_EQ(org_scan_param.get_group_by_param().add_column(str, str,is_return), OB_SUCCESS );

  /// having condtion
  c_ptr = "`item_price` > 10";
  str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
  EXPECT_EQ(org_scan_param.get_group_by_param().add_having_cond(str), OB_SUCCESS );

  /// orderby
  c_ptr = "item_price";
  str.assign((char*)c_ptr, static_cast<int32_t>(strlen(c_ptr)));
  EXPECT_EQ(org_scan_param.add_orderby_column(str),OB_SUCCESS);
}

int MutatorBuilder::build_scan_param(ObScanParam &scan_param, PageArena<char> &allocer,
                                    const int64_t table_start_version, const bool using_id, int64_t &table_pos, int64_t &prefix)
{
  int ret = OB_SUCCESS;
  scan_param.reset();

  static int64_t row_info_array[OB_MAX_TABLE_NUMBER];
  static int64_t counter = 0;
  static const int64_t ROW_INFO_CACHE_FLUSH_PERIOD = 100;
  //static const int64_t ROW_INFO_CACHE_FLUSH_PERIOD = 1000000000;
  if (0 == (counter++ % ROW_INFO_CACHE_FLUSH_PERIOD))
  {
    memset(row_info_array, 0, sizeof(row_info_array));
  }

  int64_t rand = 0;
  lrand48_r(&table_rand_seed_, &rand);
  table_pos = range_rand(0, table_num_ - 1, rand);
  const ObSchema &schema = schema_mgr_.begin()[table_pos];

  if (0 == row_info_array[table_pos]
      && 0 == (row_info_array[table_pos] = get_cur_rowkey_info_(schema, using_id, rb_array_[table_pos]->get_prefix_start())))
  {
    ret = OB_ERROR;
  }
  else
  {
    ObRange range;
    std::pair<ObString, ObString> key_range = rb_array_[table_pos]->get_rowkey4scan(row_info_array[table_pos], allocer, prefix);
    range.start_key_ = key_range.first;
    range.end_key_ = key_range.second;

    if (using_id)
    {
      scan_param.add_column(SEED_COLUMN_ID);
    }
    else
    {
      ObString column_name;
      column_name.assign_ptr((char*)SEED_COLUMN_NAME, static_cast<int32_t>(strlen(SEED_COLUMN_NAME)));
      scan_param.add_column(column_name);
    }
    if (using_id)
    {
      scan_param.add_column(CELL_NUM_COLUMN_ID);
    }
    else
    {
      ObString column_name;
      column_name.assign_ptr((char*)CELL_NUM_COLUMN_NAME, static_cast<int32_t>(strlen(CELL_NUM_COLUMN_NAME)));
      scan_param.add_column(column_name);
    }

    lrand48_r(&table_rand_seed_, &rand);
    int64_t column_num = schema.column_end() - schema.column_begin();
    for (int64_t i = 0; i < column_num; i++)
    {
      lrand48_r(&table_rand_seed_, &rand);
      int64_t column_pos = range_rand(0, column_num - 1, rand);
      //int64_t column_pos = 0;
      const ObColumnSchema &column_schema = schema.column_begin()[column_pos];
      if (using_id)
      {
        scan_param.add_column(column_schema.get_id());
      }
      else
      {
        ObString column_name;
        column_name.assign_ptr(const_cast<char*>(column_schema.get_name()), static_cast<int32_t>(strlen(column_schema.get_name()))); 
        scan_param.add_column(column_name);
      }
    }

    if (using_id)
    {
      scan_param.set(schema.get_table_id(), ObString(), range);
    }
    else
    {
      ObString table_name;
      table_name.assign_ptr(const_cast<char*>(schema.get_table_name()), static_cast<int32_t>(strlen(schema.get_table_name())));
      scan_param.set(OB_INVALID_ID, table_name, range);
    }

    ObVersionRange version_range;
    version_range.start_version_ = table_start_version;
    version_range.border_flag_.set_max_value();
    version_range.border_flag_.set_inclusive_start();
    scan_param.set_version_range(version_range);
    char* is_read_consistency_cfg = getenv("is_read_consistency");
    bool is_read_consistency = (NULL != is_read_consistency_cfg && 0 == strcmp(is_read_consistency_cfg, "true"));
    scan_param.set_is_read_consistency(is_read_consistency);
  }
  return ret;
}

    int ObScanParamLoader::get_org_scan_param(ObScanParam &param)
    {
      ObRange range;
      ObString column;
      ObString table_name;
      ObPostfixExpression comp;

      if (config_loaded_)
      {
        TBSYS_LOG(INFO, "creating org scan param");

        param.reset();

        range.reset();
        range.table_id_ = table_id_;
        range.start_key_ = range_start_;
        if (range_start_inclusive_) 
          range.border_flag_.set_inclusive_start();
        else 
          range.border_flag_.unset_inclusive_start();
        if (range_start_min_)
          range.border_flag_.set_min_value();

        range.end_key_ = range_end_;
        if (range_end_inclusive_) 
          range.border_flag_.set_inclusive_end();
        else 
          range.border_flag_.unset_inclusive_end();
        if (range_end_max_)
          range.border_flag_.set_max_value();
    
        param.set(table_id_, table_name_, range, true);
        param.set_scan_direction(scan_direction_==0?ObScanParam::FORWARD:ObScanParam::BACKWARD);
        param.set_limit_info(limit_offset_, limit_count_);

        int i = 0;
        for (i = 0; i < column_id_count_; i++)
        {
          param.add_column(column_id_[i], column_is_return_[i]);
        }

        for (i = 0; i < complex_column_id_count_; i++)
        {
          param.add_column(complex_column_id_[i],complex_column_id_[i], complex_column_is_return_[i]);
        }

        for (i = 0; i < orderby_column_id_count_; i++)
        {
          param.add_orderby_column(orderby_column_id_[i], 
              orderby_column_order_[i]==0?ObScanParam::ASC:ObScanParam::DESC);
        }

        if (where_cond_.length() > 0 &&  where_cond_.ptr() != NULL)
        {
          param.add_where_cond(where_cond_);
        }

        // groupby param
        for (i = 0; i < groupby_column_id_count_; i++)
        {
          param.get_group_by_param().add_groupby_column(groupby_column_id_[i], groupby_column_is_return_[i]);
        }
 
        for (i = 0; i < groupby_complex_column_id_count_; i++)
        {
          param.get_group_by_param().add_column(groupby_complex_column_id_[i], 
              groupby_complex_column_id_[i], 
              groupby_complex_column_is_return_[i]);
        }
 
        if (having_cond_.length() > 0 &&  having_cond_.ptr() != NULL)
        {        
          param.get_group_by_param().add_having_cond(having_cond_);
        }
        
        /// FIXME: SUM占位, 需要添加OP域
        for (i = 0; i < agg_column_id_count_; i++)
        {
          param.get_group_by_param().add_aggregate_column(agg_column_id_[i], 
              agg_column_as_name_[i], 
              (ObAggregateFuncType)agg_column_op_[i], 
              agg_column_is_return_[i]);
        }
      }
      return OB_SUCCESS;
    }