4paradigm / 4paradigm/OpenMLDB

Long window optimization effect for Real-time Feature processing is not obvious

未关闭
#2,464 6 条评论 0 个 reaction 已指派 1 人 已指派给 @zhanghaohit 在 GitHub 查看
bug execute-engine storage-engine
主要语言
C++
星标
1.7k
派生
331
平均合并
12 天 12 小时
30 天内合并 PR
1

描述

**Bug Description**
version:openmldb 0.6.1
dataset:20220814-20220821
total data:870w
Real-time Feature Processing
remove w10d: 9.5k qps
with w10d:130 qps

CREATE TABLE deployment_test_for_error(
event_id STRING,
type STRING,
p0 STRING,
p1 STRING,
p2 STRING,
p3 STRING,
p4 STRING,
p5 STRING,
p6 STRING,
p7 STRING,
event_occur_time TIMESTAMP,
report_time TIMESTAMP,
app_ver STRING,
req_id STRING,
device_name STRING,
pro_ver STRING, INDEX(KEY=event_id, TS=event_occur_time, ttl_type=absolute, ttl=60d),INDEX(KEY=(device_name,pro_ver), TS=event_occur_time, ttl_type=absolute, ttl=60d), INDEX(KEY=(app_ver,pro_ver), TS=event_occur_time, ttl_type=absolute, ttl=60d)
) OPTIONS(partitionnum=8, replicanum=3);

DEPLOY demo_lw OPTIONS(long_windows="cat_w10d:1d,ab_w10d:1d")
select
count_where(device_name, pro_ver = '12.0') OVER ab_w10m AS service_click_w10m_cnt,
count_where(device_name, pro_ver = '12.0') OVER ab_w30m AS service_click_w30m_cnt,
count_where(device_name, pro_ver = '12.0') OVER ab_w1h AS service_click_w1h_cnt,
count_where(device_name, pro_ver = '12.0') OVER ab_w3h AS service_click_w3h_cnt,
count_where(device_name, pro_ver = '12.0') OVER ab_w7h AS service_click_w7h_cnt,
count_where(device_name, pro_ver = '12.0') OVER ab_w12h AS service_click_w12h_cnt,
count_where(device_name, pro_ver = '12.0') OVER ab_w1d AS service_click_w1d_cnt,
count_where(device_name, pro_ver = '12.0') OVER ab_w10d AS service_click_w10d_cnt,
count_where(device_name, pro_ver = '6.1') OVER ab_w10m AS service_subcribe_w10m_cnt,
count_where(device_name, pro_ver = '6.1') OVER ab_w30m AS service_subcribe_w30m_cnt,
count_where(device_name, pro_ver = '6.1') OVER ab_w1h AS service_subcribe_w1h_cnt,
count_where(device_name, pro_ver = '6.1') OVER ab_w3h AS service_subcribe_w3h_cnt,
count_where(device_name, pro_ver = '6.1') OVER ab_w7h AS service_subcribe_w7h_cnt,
count_where(device_name, pro_ver = '6.1') OVER ab_w12h AS service_subcribe_w12h_cnt,
count_where(device_name, pro_ver = '6.1') OVER ab_w1d AS service_subcribe_w1d_cnt,
count_where(device_name, pro_ver = '6.1') OVER ab_w10d AS service_subcribe_w10d_cnt,
count_where(app_ver, pro_ver = '12.0') OVER cat_w10m AS category_click_w10m_cnt,
count_where(app_ver, pro_ver = '12.0') OVER cat_w30m AS category_click_w30m_cnt,
count_where(app_ver, pro_ver = '12.0') OVER cat_w1h AS category_click_w1h_cnt,
count_where(app_ver, pro_ver = '12.0') OVER cat_w3h AS category_click_w3h_cnt,
count_where(app_ver, pro_ver = '12.0') OVER cat_w7h AS category_click_w7h_cnt,
count_where(app_ver, pro_ver = '12.0') OVER cat_w12h AS category_click_w12h_cnt,
count_where(app_ver, pro_ver = '12.0') OVER cat_w1d AS category_click_w1d_cnt,
count_where(app_ver, pro_ver = '12.0') OVER cat_w10d AS category_click_w10d_cnt,
count_where(app_ver, pro_ver = '6.1') OVER cat_w10m AS category_subcribe_w10m_cnt,
count_where(app_ver, pro_ver = '6.1') OVER cat_w30m AS category_subcribe_w30m_cnt,
count_where(app_ver, pro_ver = '6.1') OVER cat_w1h AS category_subcribe_w1h_cnt,
count_where(app_ver, pro_ver = '6.1') OVER cat_w3h AS category_subcribe_w3h_cnt,
count_where(app_ver, pro_ver = '6.1') OVER cat_w7h AS category_subcribe_w7h_cnt,
count_where(app_ver, pro_ver = '6.1') OVER cat_w12h AS category_subcribe_w12h_cnt,
count_where(app_ver, pro_ver = '6.1') OVER cat_w1d AS category_subcribe_w1d_cnt,
count_where(app_ver, pro_ver = '6.1') OVER cat_w10d AS category_subcribe_w10d_cnt
FROM deployment_test_for_error
WINDOW ab_w10m AS (PARTITION BY deployment_test_for_error.device_name,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 10m PRECEDING AND CURRENT ROW),
ab_w30m AS (PARTITION BY deployment_test_for_error.device_name,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 30m PRECEDING AND CURRENT ROW),
ab_w1h AS (PARTITION BY deployment_test_for_error.device_name,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 1h PRECEDING AND CURRENT ROW),
ab_w3h AS (PARTITION BY deployment_test_for_error.device_name,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 3h PRECEDING AND CURRENT ROW),
ab_w7h AS (PARTITION BY deployment_test_for_error.device_name,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 7h PRECEDING AND CURRENT ROW),
ab_w12h AS (PARTITION BY deployment_test_for_error.device_name,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 12h PRECEDING AND CURRENT ROW),
ab_w1d AS (PARTITION BY deployment_test_for_error.device_name,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 1d PRECEDING AND CURRENT ROW),
ab_w10d AS (PARTITION BY deployment_test_for_error.device_name,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 10d PRECEDING AND CURRENT ROW),
cat_w10m AS (PARTITION BY deployment_test_for_error.app_ver,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 10m PRECEDING AND CURRENT ROW),
cat_w30m AS (PARTITION BY deployment_test_for_error.app_ver,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 30m PRECEDING AND CURRENT ROW),
cat_w1h AS (PARTITION BY deployment_test_for_error.app_ver,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 1h PRECEDING AND CURRENT ROW),
cat_w3h AS (PARTITION BY deployment_test_for_error.app_ver,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 3h PRECEDING AND CURRENT ROW),
cat_w7h AS (PARTITION BY deployment_test_for_error.app_ver,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 7h PRECEDING AND CURRENT ROW),
cat_w12h AS (PARTITION BY deployment_test_for_error.app_ver,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 12h PRECEDING AND CURRENT ROW),
cat_w1d AS (PARTITION BY deployment_test_for_error.app_ver,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 1d PRECEDING AND CURRENT ROW),
cat_w10d AS (PARTITION BY deployment_test_for_error.app_ver,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 10d PRECEDING AND CURRENT ROW);

DEPLOY demo
select
count_where(device_name, pro_ver = '12.0') OVER ab_w10m AS service_click_w10m_cnt,
count_where(device_name, pro_ver = '12.0') OVER ab_w30m AS service_click_w30m_cnt,
count_where(device_name, pro_ver = '12.0') OVER ab_w1h AS service_click_w1h_cnt,
count_where(device_name, pro_ver = '12.0') OVER ab_w3h AS service_click_w3h_cnt,
count_where(device_name, pro_ver = '12.0') OVER ab_w7h AS service_click_w7h_cnt,
count_where(device_name, pro_ver = '12.0') OVER ab_w12h AS service_click_w12h_cnt,
count_where(device_name, pro_ver = '12.0') OVER ab_w1d AS service_click_w1d_cnt,
count_where(device_name, pro_ver = '12.0') OVER ab_w10d AS service_click_w10d_cnt,
count_where(device_name, pro_ver = '6.1') OVER ab_w10m AS service_subcribe_w10m_cnt,
count_where(device_name, pro_ver = '6.1') OVER ab_w30m AS service_subcribe_w30m_cnt,
count_where(device_name, pro_ver = '6.1') OVER ab_w1h AS service_subcribe_w1h_cnt,
count_where(device_name, pro_ver = '6.1') OVER ab_w3h AS service_subcribe_w3h_cnt,
count_where(device_name, pro_ver = '6.1') OVER ab_w7h AS service_subcribe_w7h_cnt,
count_where(device_name, pro_ver = '6.1') OVER ab_w12h AS service_subcribe_w12h_cnt,
count_where(device_name, pro_ver = '6.1') OVER ab_w1d AS service_subcribe_w1d_cnt,
count_where(device_name, pro_ver = '6.1') OVER ab_w10d AS service_subcribe_w10d_cnt,
count_where(app_ver, pro_ver = '12.0') OVER cat_w10m AS category_click_w10m_cnt,
count_where(app_ver, pro_ver = '12.0') OVER cat_w30m AS category_click_w30m_cnt,
count_where(app_ver, pro_ver = '12.0') OVER cat_w1h AS category_click_w1h_cnt,
count_where(app_ver, pro_ver = '12.0') OVER cat_w3h AS category_click_w3h_cnt,
count_where(app_ver, pro_ver = '12.0') OVER cat_w7h AS category_click_w7h_cnt,
count_where(app_ver, pro_ver = '12.0') OVER cat_w12h AS category_click_w12h_cnt,
count_where(app_ver, pro_ver = '12.0') OVER cat_w1d AS category_click_w1d_cnt,
count_where(app_ver, pro_ver = '12.0') OVER cat_w10d AS category_click_w10d_cnt,
count_where(app_ver, pro_ver = '6.1') OVER cat_w10m AS category_subcribe_w10m_cnt,
count_where(app_ver, pro_ver = '6.1') OVER cat_w30m AS category_subcribe_w30m_cnt,
count_where(app_ver, pro_ver = '6.1') OVER cat_w1h AS category_subcribe_w1h_cnt,
count_where(app_ver, pro_ver = '6.1') OVER cat_w3h AS category_subcribe_w3h_cnt,
count_where(app_ver, pro_ver = '6.1') OVER cat_w7h AS category_subcribe_w7h_cnt,
count_where(app_ver, pro_ver = '6.1') OVER cat_w12h AS category_subcribe_w12h_cnt,
count_where(app_ver, pro_ver = '6.1') OVER cat_w1d AS category_subcribe_w1d_cnt,
count_where(app_ver, pro_ver = '6.1') OVER cat_w10d AS category_subcribe_w10d_cnt
FROM deployment_test_for_error
WINDOW ab_w10m AS (PARTITION BY deployment_test_for_error.device_name,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 10m PRECEDING AND CURRENT ROW),
ab_w30m AS (PARTITION BY deployment_test_for_error.device_name,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 30m PRECEDING AND CURRENT ROW),
ab_w1h AS (PARTITION BY deployment_test_for_error.device_name,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 1h PRECEDING AND CURRENT ROW),
ab_w3h AS (PARTITION BY deployment_test_for_error.device_name,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 3h PRECEDING AND CURRENT ROW),
ab_w7h AS (PARTITION BY deployment_test_for_error.device_name,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 7h PRECEDING AND CURRENT ROW),
ab_w12h AS (PARTITION BY deployment_test_for_error.device_name,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 12h PRECEDING AND CURRENT ROW),
ab_w1d AS (PARTITION BY deployment_test_for_error.device_name,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 1d PRECEDING AND CURRENT ROW),
ab_w10d AS (PARTITION BY deployment_test_for_error.device_name,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 10d PRECEDING AND CURRENT ROW),
cat_w10m AS (PARTITION BY deployment_test_for_error.app_ver,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 10m PRECEDING AND CURRENT ROW),
cat_w30m AS (PARTITION BY deployment_test_for_error.app_ver,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 30m PRECEDING AND CURRENT ROW),
cat_w1h AS (PARTITION BY deployment_test_for_error.app_ver,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 1h PRECEDING AND CURRENT ROW),
cat_w3h AS (PARTITION BY deployment_test_for_error.app_ver,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 3h PRECEDING AND CURRENT ROW),
cat_w7h AS (PARTITION BY deployment_test_for_error.app_ver,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 7h PRECEDING AND CURRENT ROW),
cat_w12h AS (PARTITION BY deployment_test_for_error.app_ver,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 12h PRECEDING AND CURRENT ROW),
cat_w1d AS (PARTITION BY deployment_test_for_error.app_ver,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 1d PRECEDING AND CURRENT ROW),
cat_w10d AS (PARTITION BY deployment_test_for_error.app_ver,deployment_test_for_error.pro_ver ORDER BY deployment_test_for_error.event_occur_time ROWS_RANGE BETWEEN 10d PRECEDING AND CURRENT ROW);

`10.97.152.110:6527/> select * from __INTERNAL_DB.PRE_AGG_META_INFO;
------------------------------------------------------------- -------------- --------- --------------------------- ------------- ------------- ---------------------- ------------------ ------------- ------------
aggr_table aggr_db base_db base_table aggr_func aggr_col partition_cols order_by_col bucket_size filter_col
------------------------------------------------------------- -------------- --------- --------------------------- ------------- ------------- ---------------------- ------------------ ------------- ------------
pre_demo_db_demo_lw_cat_w10d_count_where_app_ver_pro_ver __PRE_AGG_DB demo_db deployment_test_for_error count_where app_ver app_ver,pro_ver event_occur_time 1d pro_ver
pre_demo_db_demo_lw_cat_w10d_count_where_app_ver_pro_ver __PRE_AGG_DB demo_db deployment_test_for_error count_where app_ver app_ver,pro_ver event_occur_time 1d pro_ver
pre_demo_db_demo_lw_ab_w10d_count_where_device_name_pro_ver __PRE_AGG_DB demo_db deployment_test_for_error count_where device_name device_name,pro_ver event_occur_time 1d pro_ver
pre_demo_db_demo_lw_ab_w10d_count_where_device_name_pro_ver __PRE_AGG_DB demo_db deployment_test_for_error count_where device_name device_name,pro_ver event_occur_time 1d pro_ver `

**Expected Behavior**

**Steps to Reproduce**

1.
2.
3.
4.

贡献指南

打开贡献指南

调研方向

该 issue 报告了在 OpenMLDB 实时 feature 处理中使用长窗口 (w10d) 时的性能下降。检查带有 long_windows 选项的 SQL deployment,并与不带该选项的 deployment 进行比较。查看 query engine 中的窗口优化逻辑,该逻辑可能位于 src/engine 或类似目录中。运行提供的 SQL 以复现 QPS 差异,然后进行 profiling,以识别长窗口处理中的瓶颈。

由索引模型根据 Issue 内容生成。

评估

技术栈
sql
领域
databases, performance
Issue 类型
缺陷
难度
4/5
预计耗时
3-5 天
活跃度
停滞
描述清晰度
基本清楚
新手友好度
35/100

把新 issue 发到你的邮箱

精选适合新手参与的 GitHub issue 摘要。