• 2022-07-05 stonedb的子查询处理解析

    摘要:

    记录stonedb的子查询处理

    核心函数:

    ParameterizedFilter::ApplyDescriptor

    1. void ParameterizedFilter::ApplyDescriptor(int desc_number, int64_t limit)
    2. // desc_number = -1 => switch off the rough part
    3. {
    4.   Descriptor &desc = descriptors[desc_number];
    5.   if (desc.op == common::Operator::O_TRUE) {
    6.     desc.done = true;
    7.     return;
    8.   }
    9.   if (desc.op == common::Operator::O_FALSE) {
    10.     mind->Empty();
    11.     desc.done = true;
    12.     return;
    13.   }
    14.  
    15.   DimensionVector dims(mind->NoDimensions());
    16.   desc.DimensionUsed(dims);
    17.   mind->MarkInvolvedDimGroups(dims);  // create iterators on whole groups (important for
    18.                                       // multidimensional updatable iterators)
    19.   int no_dims = dims.NoDimsUsed();
    20.   if (no_dims == 0 && !desc.IsDeterministic()) dims.SetAll();
    21.   // Check the easy case (one-dim, parallelizable)
    22.   int one_dim = -1;
    23.   common::RSValue *rf = NULL;
    24.   if (no_dims == 1) {
    25.     for (int i = 0; i < mind->NoDimensions(); i++) {
    26.       if (dims[i]) {
    27.         if (mind->GetFilter(i)) one_dim = i;  // exactly one filter (non-join or join with forgotten dims)
    28.         break;
    29.       }
    30.     }
    31.   }
    32.   if (one_dim != -1)
    33.     rf = rough_mind->GetLocalDescFilter(one_dim, desc_number,
    34.                                         true);  // "true" here means that we demand an existing local rough
    35.                                                 // filter
    36.  
    37.   int packs_no = (int)((mind->OrigSize(one_dim) + ((1 << mind->NoPower()) - 1)) >> mind->NoPower());
    38.   int pack_all = rough_mind->NoPacks(one_dim);
    39.   int pack_some = 0;
    40.   for (int b = 0; b < pack_all; b++) {
    41.     if (rough_mind->GetPackStatus(one_dim, b) != common::RSValue::RS_NONE) pack_some++;
    42.   }
    43.   MIUpdatingIterator mit(mind, dims);
    44.   desc.CopyDesCond(mit);
    45.   if (desc.EvaluateOnIndex(mit, limit) == common::ErrorCode::SUCCESS) {
    46.     rccontrol.lock(mind->m_conn->GetThreadID())
    47.         << "EvaluateOnIndex done, desc number " << desc_number << system::unlock;
    48.   } else {
    49.     int poolsize = rceng->query_thread_pool.size();
    50.     if ((stonedb_sysvar_threadpoolsize > 0) && (packs_no / poolsize > 0) && !desc.IsType_Subquery() &&
    51.         !desc.ExsitTmpTable()) {
    52.       int step = 0;
    53.       int task_num = 0;
    54.       /*Partition task slice*/
    55.       if (pack_some <= poolsize) {
    56.         task_num = poolsize;
    57.       } else {
    58.         step = pack_some / poolsize;
    59.         task_num = packs_no / step;
    60.       }
    61.       int mod = packs_no % task_num;
    62.       int num = packs_no / task_num;
    63.  
    64.       desc.InitParallel(task_num, mit);
    65.  
    66.       std::vector<MultiIndex> mis;
    67.       mis.reserve(task_num);
    68.  
    69.       std::vector<MIUpdatingIterator> taskIterator;
    70.       taskIterator.reserve(task_num);
    71.  
    72.       for (int i = 0; i < task_num; ++i) {
    73.         auto &mi = mis.emplace_back(*mind, true);
    74.         int pstart = ((i == 0) ? 0 : mod + i * num);
    75.         int pend = mod + (i + 1) * num - 1;
    76.  
    77.         auto &mii = taskIterator.emplace_back(&mi, dims);
    78.         mii.SetTaskNum(task_num);
    79.         mii.SetTaskId(i);
    80.         mii.SetNoPacksToGo(pend);
    81.         mii.RewindToPack(pstart);
    82.       }
    83.  
    84.       utils::result_set<void> res;
    85.       for (int i = 0; i < task_num; ++i) {
    86.         res.insert(rceng->query_thread_pool.add_task(&ParameterizedFilter::TaskProcessPacks, this, &taskIterator[i],
    87.                                                      current_tx, rf, &dims, desc_number, limit, one_dim));
    88.       }
    89.       res.get_all_with_except();
    90.  
    91.       if (mind->m_conn->Killed()) throw common::KilledException("catch thread pool Exception: TaskProcessPacks");
    92.       mind->UpdateNoTuples();
    93.  
    94.     } else {
    95.       common::RSValue cur_roughval;
    96.       uint64_t passed = 0;
    97.       int pack = -1;
    98.       while (mit.IsValid()) {
    99.         if (limit != -1 && rf) {  // rf - not null if there is one dim only
    100.                                   // (otherwise packs make no sense)
    101.           if (passed >= (uint64_t)limit) {
    102.             mit.ResetCurrentPack();
    103.             mit.NextPackrow();
    104.             continue;
    105.           }
    106.           if (mit.PackrowStarted()) {
    107.             if (pack != -1) passed += mit.NoOnesUncommited(pack);
    108.             pack = mit.GetCurPackrow(one_dim);
    109.           }
    110.         }
    111.  
    112.         if (rf && mit.GetCurPackrow(one_dim) >= 0)
    113.           cur_roughval = rf[mit.GetCurPackrow(one_dim)];
    114.         else
    115.           cur_roughval = common::RSValue::RS_SOME;
    116.  
    117.         if (cur_roughval == common::RSValue::RS_NONE) {
    118.           mit.ResetCurrentPack();
    119.           mit.NextPackrow();
    120.         } else if (cur_roughval == common::RSValue::RS_ALL) {
    121.           mit.NextPackrow();
    122.         } else {
    123.           // common::RSValue::RS_SOME or common::RSValue::RS_UNKNOWN
    124.           desc.EvaluatePack(mit);
    125.         }
    126.         if (mind->m_conn->Killed()) throw common::KilledException();
    127.       }
    128.       mit.Commit();
    129.     }
    130.   }
    131.   desc.done = true;
    132.   if (one_dim != -1 && mind->GetFilter(one_dim)) {  // update global rough part
    133.     Filter *f = mind->GetFilter(one_dim);
    134.     for (int p = 0; p < rough_mind->NoPacks(one_dim); p++)
    135.       if (f->IsEmpty(p)) {
    136.         rough_mind->SetPackStatus(one_dim, p, common::RSValue::RS_NONE);
    137.       }
    138.   }
    139.   desc.UpdateVCStatistics();
    140.   return;
    141. }

    子查询处理：

    1.       common::RSValue cur_roughval;
    2.       uint64_t passed = 0;
    3.       int pack = -1;
    4.       while (mit.IsValid()) {
    5.         if (limit != -1 && rf) {  // rf - not null if there is one dim only
    6.                                   // (otherwise packs make no sense)
    7.           if (passed >= (uint64_t)limit) {
    8.             mit.ResetCurrentPack();
    9.             mit.NextPackrow();
    10.             continue;
    11.           }
    12.           if (mit.PackrowStarted()) {
    13.             if (pack != -1) passed += mit.NoOnesUncommited(pack);
    14.             pack = mit.GetCurPackrow(one_dim);
    15.           }
    16.         }
    17.  
    18.         if (rf && mit.GetCurPackrow(one_dim) >= 0)
    19.           cur_roughval = rf[mit.GetCurPackrow(one_dim)];
    20.         else
    21.           cur_roughval = common::RSValue::RS_SOME;
    22.  
    23.         if (cur_roughval == common::RSValue::RS_NONE) {
    24.           mit.ResetCurrentPack();
    25.           mit.NextPackrow();
    26.         } else if (cur_roughval == common::RSValue::RS_ALL) {
    27.           mit.NextPackrow();
    28.         } else {
    29.           // common::RSValue::RS_SOME or common::RSValue::RS_UNKNOWN
    30.           desc.EvaluatePack(mit);
    31.         }
    32.         if (mind->m_conn->Killed()) throw common::KilledException();
    33.       }
    34.       mit.Commit();

    调用堆栈:

    1. (gdb) bt
    2. #0  stonedb::core::ParameterizedFilter::ApplyDescriptor (this=0x7fa8d404e7a0, desc_number=0, limit=-1)
    3.     at /root/work/stonedb/storage/stonedb/core/parameterized_filter.cpp:1306
    4. #1  0x0000000001d4ef85 in stonedb::core::ParameterizedFilter::UpdateMultiIndex (this=0x7fa8d404e7a0, count_only=false, limit=-1)
    5.     at /root/work/stonedb/storage/stonedb/core/parameterized_filter.cpp:1053
    6. #2  0x00000000019e95e0 in stonedb::core::Query::Preexecute (this=0x7fa8d4027780, qu=..., sender=0x0, display_now=false)
    7.     at /root/work/stonedb/storage/stonedb/core/query.cpp:776
    8. #3  0x0000000001d9172e in stonedb::dbhandler::StonedbHandler::set_cond_iter (this=0x7fa8d4034910)
    9.     at /root/work/stonedb/storage/stonedb/handler/stonedb_handler.cpp:1315
    10. #4  0x0000000001d92482 in stonedb::dbhandler::StonedbHandler::cond_push (this=0x7fa8d4034910, a_cond=0x7fa8d4044f20)
    11.     at /root/work/stonedb/storage/stonedb/handler/stonedb_handler.cpp:1399
    12. #5  0x000000000154590e in make_join_select (join=0x7fa8d4043930, cond=0x7fa8d404b710) at /root/work/stonedb/sql/sql_optimizer.cc:7806
    13. #6  0x0000000001533ae4 in JOIN::optimize (this=0x7fa8d4043930, part=0 '\000') at /root/work/stonedb/sql/sql_optimizer.cc:513
    14. #7  0x00000000014c720c in subselect_single_select_engine::exec (this=0x7fa8d40093a0) at /root/work/stonedb/sql/item_subselect.cc:2747
    15. #8  0x00000000014c07c5 in Item_subselect::exec (this=0x7fa8d404b800) at /root/work/stonedb/sql/item_subselect.cc:642
    16. #9  0x00000000014c26d6 in Item_exists_subselect::val_int (this=0x7fa8d404b800) at /root/work/stonedb/sql/item_subselect.cc:1269
    17. #10 0x000000000125338c in eval_const_cond (cond=0x7fa8d404b800) at /root/work/stonedb/sql/item_func.cc:76
    18. #11 0x0000000001548155 in internal_remove_eq_conds (thd=0x41658a0, cond=0x7fa8d404b800, cond_value=0x7fa90140b6dc, part=1 '\001')
    19.     at /root/work/stonedb/sql/sql_optimizer.cc:8681
    20. #12 0x0000000001547c68 in internal_remove_eq_conds (thd=0x41658a0, cond=0x7fa8d40089e0, cond_value=0x4167e38, part=1 '\001')
    21.     at /root/work/stonedb/sql/sql_optimizer.cc:8552
    22. #13 0x0000000001548516 in remove_eq_conds (thd=0x41658a0, cond=0x7fa8d40089e0, cond_value=0x4167e38, part=1 '\001') at /root/work/stonedb/sql/sql_optimizer.cc:8774
    23. #14 0x00000000015479da in optimize_cond (thd=0x41658a0, conds=0x7fa8d40089e0, cond_equal=0x7fa8d404c008, join_list=0x4167ee8, build_equalities=true, 
    24.     cond_value=0x4167e38, part=1 '\001') at /root/work/stonedb/sql/sql_optimizer.cc:8507
    25. #15 0x00000000015329cf in JOIN::optimize (this=0x7fa8d404bca0, part=1 '\001') at /root/work/stonedb/sql/sql_optimizer.cc:246
    26. #16 0x00000000019b7c3e in stonedb::core::optimize_select (thd=0x41658a0, tables=0x7fa8d4007ac0, wild_num=0, fields=..., conds=0x7fa8d40089e0, og_num=2, 
    27.     order=0x7fa8d404bb80, group=0x7fa8d404ba20, having=0x0, select_options=2147748608, result=0x7fa8d404bc68, unit=0x4167720, select_lex=0x4167d68, 
    28.     optimize_after_sdb=@0x7fa90140c094: 1, free_join=@0x7fa90140c098: 1) at /root/work/stonedb/storage/stonedb/core/engine_execute.cpp:339
    29. #17 0x00000000019b7588 in stonedb::core::Engine::HandleSelect (this=0x41b1150, thd=0x41658a0, lex=0x4167660, result=@0x7fa90140c0a0: 0x7fa8d404bc68, 
    30.     setup_tables_done_option=0, res=@0x7fa90140c09c: 0, optimize_after_sdb=@0x7fa90140c094: 1, sdb_free_join=@0x7fa90140c098: 1, with_insert=0)
    31.     at /root/work/stonedb/storage/stonedb/core/engine_execute.cpp:223
    32. #18 0x0000000001aacee3 in stonedb::dbhandler::SDB_HandleSelect (thd=0x41658a0, lex=0x4167660, result=@0x7fa90140c0a0: 0x7fa8d404bc68, setup_tables_done_option=0, 
    33.     res=@0x7fa90140c09c: 0, optimize_after_sdb=@0x7fa90140c094: 1, sdb_free_join=@0x7fa90140c098: 1, with_insert=0)
    34.     at /root/work/stonedb/storage/stonedb/handler/ha_rcengine.cpp:82
    35. #19 0x000000000137b090 in execute_sqlcom_select (thd=0x41658a0, all_tables=0x7fa8d4007ac0)
    36. #20 0x0000000001373ea1 in mysql_execute_command (thd=0x41658a0)
    37. #21 0x000000000137daec in mysql_parse (thd=0x41658a0, 
    38.     rawbuf=0x7fa8d40073f0 "select\n    o_carrier_id,\n    count(*) as order_count\nfrom\n    orders\nwhere\n    o_entry_d >= date '1993-07-01'\n    and o_entry_d < date '2022-07-06' + interval '3' month\n    and exists (\n        select"..., length=374, parser_state=0x7fa90140d170)
    39. #22 0x000000000137109d in dispatch_command (command=COM_QUERY, thd=0x41658a0, 
    40.     packet=0x4168bf1 "select\n    o_carrier_id,\n    count(*) as order_count\nfrom\n    orders\nwhere\n    o_entry_d >= date '1993-07-01'\n    and o_entry_d < date '2022-07-06' + interval '3' month\n    and exists (\n        select"..., packet_length=374)
    41. #23 0x00000000013701d7 in do_command (thd=0x41658a0)
    42. #24 0x000000000133be43 in do_handle_one_connection (thd_arg=0x41658a0) at /root/work/stonedb/sql/sql_connect.cc:982
    43. #25 0x000000000133b9a6 in handle_one_connection (arg=0x41658a0) at /root/work/stonedb/sql/sql_connect.cc:898
    44. #26 0x00000000019530a2 in pfs_spawn_thread (arg=0x4842510) at /root/work/stonedb/storage/perfschema/pfs.cc:1860
    45. #27 0x00007faa73470ea5 in start_thread () from /lib64/libpthread.so.0
    46. #28 0x00007faa71f7cb0d in clone () from /lib64/libc.so.6

    1. (gdb) p mit
    2. $6 = {
    3.   <stonedb::core::MIIterator> = {
    4.     _vptr.MIIterator = 0x24b1578 <vtable for stonedb::core::MIUpdatingIterator+16>, 
    5.     it = std::vector of length 1, capacity 1 = {0x7fa8d404e9b0}, 
    6.     dg = std::vector of length 1, capacity 1 = {0x7fa8d401aa80}, 
    7.     it_for_dim = 0x7fa8d4047b60, 
    8.     mind = 0x7fa8d404e900, 
    9.     mind_created_locally = false, 
    10.     no_dims = 1, 
    11.     dimensions = {
    12.       v = std::vector<bool> of length 1, capacity 64 = {1}
    13.     }, 
    14.     cur_pos = 0x7fa8d4046ee0, 
    15.     cur_pack = 0x7fa8d4027980, 
    16.     valid = true, 
    17.     omitted_factor = 1, 
    18.     no_obj = 100000, 
    19.     p_power = 16, 
    20.     pack_size_left = 65536, 
    21.     next_pack_started = true, 
    22.     mii_type = stonedb::core::MIIterator::MII_NORMAL, 
    23.     po = std::vector of length 0, capacity 0, 
    24.     one_filter_dim = 0, 
    25.     one_filter_it = 0x7fa8d404e9b0, 
    26.     TaskId = 0, 
    27.     TasksNum = 1
    28.   }, 
    29.   members of stonedb::core::MIUpdatingIterator: 
    30.   pack_power = 16, 
    31.   changed = false, 
    32. Python Exception <class 'gdb.error'> There is no member or method named _M_head_impl.: 
    33.   multi_dim_filter = , 
    34.   multi_filter_pos = 0, 
    35.   multi_filter_pack_start = 0
    36. }
    1. (gdb) ptype mit
    2. type = class stonedb::core::MIUpdatingIterator : public stonedb::core::MIIterator {
    3.   private:
    4.     uint32_t pack_power;
    5.     bool changed;
    6.     std::unique_ptr<stonedb::core::Filter, std::default_delete<stonedb::core::Filter> > multi_dim_filter;
    7.     int64_t multi_filter_pos;
    8.     int64_t multi_filter_pack_start;
    9.  
    10.   public:
    11.     MIUpdatingIterator(stonedb::core::MultiIndex *, stonedb::core::DimensionVector &);
    12.     MIUpdatingIterator(const stonedb::core::MIUpdatingIterator &);
    13.     MIUpdatingIterator(stonedb::core::MIUpdatingIterator &);
    14.     stonedb::core::MIUpdatingIterator & operator=(const stonedb::core::MIUpdatingIterator &);
    15.     ~MIUpdatingIterator(int);
    16.     void ResetCurrent(void);
    17.     void ResetCurrentPack(void);
    18.     void Commit(bool);
    19.     void Rewind(void);
    20.     stonedb::core::MIUpdatingIterator & operator++(void);
    21.     virtual void NextPackrow(void);
    22.     void RewindToRow(int64_t);
    23.     virtual bool RewindToPack(int);
    24.     int NoOnesUncommited(uint);
    25.     void UpdateNoTuples(void);
    26.     int SingleFilterDim(void) const;
    27.     stonedb::core::Filter * NewPackFilter(int);
    28.     bool SwapPackFilter(int, stonedb::core::Filter *);
    29.     void OrPackFilter(int, stonedb::core::Filter *);
    30. }

  • 相关阅读:
    今天是七夕，来看看程序员的土味情话。
    jwt(json web token)
    沉睡者IT - 说几个2022年网络上比较好赚钱的创业项目
    OpenCV教程：cv2图像逻辑运算
    Abaqus血管支架仿真攻略之几何创建与网格划分
    基于SSM的田径运动会成绩管理系统的设计与实现
    刷题日常计~JS①
    Docker+K8s基础(重要知识点总结)
    搞什么副业可以月入过万？你们觉得抖音小店做起来能赚多少钱?
    python数据容器——列表
  • 原文地址：https://blog.csdn.net/adofsauron/article/details/125617509