4paradigm / 4paradigm/OpenMLDB

Potential performance challenges for index creation in big data scenarios

未关闭
#3,840 0 条评论 0 个 reaction 已指派 1 人 已被 @aceforeverd 认领 在 GitHub 查看
bug
主要语言
C++
星标
1.7k
派生
331
平均合并
12 天 12 小时
30 天内合并 PR
1

描述

**Bug Description**

While conducting memory tests on OpenMLDB, I created an index on the "app" column of the TalkingData train dataset. However, I soon realized that creating an index on the 'app' column held no practical significance due to severe data skew in the "app" column. Nonetheless, during the process, I observed an abnormal phenomenon: the index creation task took over two days to complete, far beyond the expected time. This observation may indicate potential issues that could arise when creating indexes in scenarios involving large amounts of data.

![image](https://github.com/4paradigm/OpenMLDB/assets/162977423/b28cfba0-dfff-4547-971b-408b70f17086)

**Expected Behavior**
The index creation task should be able to complete within a few minutes or a few tens of minutes.

**Steps to Reproduce**
1. get dataset from here: https://www.kaggle.com/c/talkingdata-adtracking-fraud-detection/data, download train.csv;
2. create db and table (test_talking_mem);
3. `LOAD DATA INFILE 'file:///tmp/train.csv' INTO TABLE test_talking_mem OPTIONS(delimiter = ',', mode = 'append');`
4. `CREATE INDEX idx_app ON test_talking_mem (app);`

**Some potentially useful information**

1. Table info

Table info

```json
{
"name": "test_talking_mem",
"table_partition":
[
{
"pid": 0,
"partition_meta":
[
{
"endpoint": "m7-pce-hdp03:10922",
"is_leader": true,
"offset": 50113235,
"record_cnt": 23044430,
"record_byte_size": 5653552902,
"tablet_has_partition": true,
"diskused": 8434468909
}
],
"term_offset":
[
{
"term": 1,
"offset": 0
}
],
"record_cnt": 23044430,
"record_byte_size": 5653552902,
"diskused": 8434468909
},
{
"pid": 1,
"partition_meta":
[
{
"endpoint": "m7-pce-hdp02:10921",
"is_leader": true,
"offset": 24437207,
"record_cnt": 23070342,
"record_byte_size": 2466003867,
"tablet_has_partition": true,
"diskused": 2609541904
}
],
"term_offset":
[
{
"term": 1,
"offset": 0
}
],
"record_cnt": 23070342,
"record_byte_size": 2466003867,
"diskused": 2609541904
},
{
"pid": 2,
"partition_meta":
[
{
"endpoint": "m7-pce-hdp03:10922",
"is_leader": true,
"offset": 40330630,
"record_cnt": 23083002,
"record_byte_size": 4280679623,
"tablet_has_partition": true,
"diskused": 3809344191
}
],
"term_offset":
[
{
"term": 1,
"offset": 0
}
],
"record_cnt": 23083002,
"record_byte_size": 4280679623,
"diskused": 3809344191
},
{
"pid": 3,
"partition_meta":
[
{
"endpoint": "m7-pce-hdp02:10921",
"is_leader": true,
"offset": 29068056,
"record_cnt": 24328781,
"record_byte_size": 2979013120,
"tablet_has_partition": true,
"diskused": 3058134341
}
],
"term_offset":
[
{
"term": 1,
"offset": 0
}
],
"record_cnt": 24328781,
"record_byte_size": 2979013120,
"diskused": 3058134341
},
{
"pid": 4,
"partition_meta":
[
{
"endpoint": "m7-pce-hdp03:10922",
"is_leader": true,
"offset": 42039005,
"record_cnt": 22953742,
"record_byte_size": 4655333631,
"tablet_has_partition": true,
"diskused": 6806149618
}
],
"term_offset":
[
{
"term": 1,
"offset": 0
}
],
"record_cnt": 22953742,
"record_byte_size": 4655333631,
"diskused": 6806149618
},
{
"pid": 5,
"partition_meta":
[
{
"endpoint": "m7-pce-hdp02:10921",
"is_leader": true,
"offset": 50715685,
"record_cnt": 22860283,
"record_byte_size": 5464521228,
"tablet_has_partition": true,
"diskused": 4573758877
}
],
"term_offset":
[
{
"term": 1,
"offset": 0
}
],
"record_cnt": 22860283,
"record_byte_size": 5464521228,
"diskused": 4573758877
},
{
"pid": 6,
"partition_meta":
[
{
"endpoint": "m7-pce-hdp03:10922",
"is_leader": true,
"offset": 35192075,
"record_cnt": 22315199,
"record_byte_size": 3697029230,
"tablet_has_partition": true,
"diskused": 3359285519
}
],
"term_offset":
[
{
"term": 1,
"offset": 0
}
],
"record_cnt": 22315199,
"record_byte_size": 3697029230,
"diskused": 3359285519
},
{
"pid": 7,
"partition_meta":
[
{
"endpoint": "m7-pce-hdp02:10921",
"is_leader": true,
"offset": 43004593,
"record_cnt": 23248111,
"record_byte_size": 4602049655,
"tablet_has_partition": true,
"diskused": 6754254073
}
],
"term_offset":
[
{
"term": 1,
"offset": 0
}
],
"record_cnt": 23248111,
"record_byte_size": 4602049655,
"diskused": 6754254073
}
],
"tid": 47,
"partition_num": 8,
"replica_num": 1,
"compress_type": "kNoCompress",
"column_desc":
[
{
"name": "ip",
"data_type": "kVarchar",
"not_null": false
},
{
"name": "app",
"data_type": "kInt",
"not_null": false
},
{
"name": "device",
"data_type": "kInt",
"not_null": false
},
{
"name": "os",
"data_type": "kInt",
"not_null": false
},
{
"name": "channel",
"data_type": "kInt",
"not_null": false
},
{
"name": "click_time",
"data_type": "kTimestamp",
"not_null": false
},
{
"name": "is_attributed",
"data_type": "kInt",
"not_null": false
}
],
"column_key":
[
{
"index_name": "INDEX_0_1711463208",
"col_name":
[
"ip"
],
"ttl":
{
"ttl_type": "kAbsoluteTime",
"abs_ttl": 0
}
},
{
"index_name": "idx_app",
"col_name":
[
"app"
],
"ttl":
{
"ttl_type": "kAbsoluteTime",
"abs_ttl": 0
}
}
],
"db": "mem_test",
"storage_mode": "kMemory"
}
```

2. data skew

https://www.kaggle.com/c/talkingdata-adtracking-fraud-detection/data?select=train.csv
![image](https://github.com/4paradigm/OpenMLDB/assets/162977423/89308866-3a7d-4245-a7e7-68f11b7dc4a5)

贡献指南

打开贡献指南

评估

这个 Issue 还没有评估数据。

把新 issue 发到你的邮箱

精选适合新手参与的 GitHub issue 摘要。