pingcap / pingcap/tidb

support rewrite SQL to remove aggregation for distinct when there is a UNION in SQL

Open
#67,986 0 comments 0 reactions 1 assignee Claimed by @guo-shaoge View on GitHub
plan-rewrite report/customer sig/planner type/enhancement
Dominant language
Go
Stars
40.5k
Forks
6.2k
PR merge metrics
PR metrics pending

Description

## Enhancement
There are two enhancements hidden in the following sql:
1. we should rewrite the common part in the sql as CTE to avoid duplicated heavy join computation. And ther e is already a issue for this problem: https://github.com/pingcap/tidb/issues/67746
2. we should remove `StreamAgg_72` for distinction automatically when there is a `UNION`, which already include the distinct semantics
```
DROP TABLE IF EXISTS t0;
DROP TABLE IF EXISTS t1;
DROP TABLE IF EXISTS t2;
DROP TABLE IF EXISTS t3;
CREATE TABLE t0 (
c0 BIGINT NOT NULL,
c1 BIGINT NOT NULL,
c2 VARCHAR(32) NOT NULL,
c3 TINYINT NOT NULL,
KEY idx_t0_c1_c2_c3 (c1, c2, c3),
KEY idx_t0_c2_c0 (c2, c0)
);
CREATE TABLE t1 (
c0 BIGINT NOT NULL,
c1 VARCHAR(32) NOT NULL,
c2 BIGINT NOT NULL,
c3 INT NOT NULL,
c4 INT NOT NULL,
KEY idx_t1_c0_c1 (c0, c1),
KEY idx_t1_c1_c3_c4 (c1, c3, c4)
);
CREATE TABLE t2 (
c0 BIGINT NOT NULL,
c1 VARCHAR(32) NOT NULL,
c2 INT NOT NULL,
KEY idx_t2_c0_c1 (c0, c1),
KEY idx_t2_c1_c2 (c1, c2)
);
CREATE TABLE t3 (
c0 BIGINT NOT NULL,
c1 BIGINT NOT NULL,
c2 VARCHAR(32) NOT NULL,
KEY idx_t3_c2_c0 (c2, c0)
);

-- original SQL
SELECT
c0,
c1
FROM
t3
WHERE
c0 IN (
SELECT
c0
FROM
(
(
SELECT DISTINCT t0.c0 AS c0
FROM t0
JOIN t1
ON t1.c0 = t0.c1
AND t1.c1 = t0.c2
JOIN t2
ON t2.c0 = t1.c2
AND t2.c1 = t1.c1
WHERE t1.c3 >= 20260417
AND t1.c4 <= 20260417
AND t0.c2 = '10798'
AND t0.c3 = 0
AND t2.c2 >= 20260417
AND t2.c2 <= 99991231
)
UNION
(
SELECT DISTINCT t0.c1 AS c0
FROM t0
JOIN t1
ON t1.c0 = t0.c1
AND t1.c1 = t0.c2
JOIN t2
ON t2.c0 = t1.c2
AND t2.c1 = t1.c1
WHERE t1.c3 >= 20260417
AND t1.c4 <= 20260417
AND t0.c2 = '10798'
AND t0.c3 = 0
AND t2.c2 >= 20260417
AND t2.c2 <= 99991231
)
) s
)
AND c2 = '10798';

-- optimized SQL
WITH z0 AS (
SELECT
t0.c0,
t0.c1
FROM t0
JOIN t1
ON t1.c0 = t0.c1
AND t1.c1 = t0.c2
JOIN t2
ON t2.c0 = t1.c2
AND t2.c1 = t1.c1
WHERE t1.c3 >= 20260417
AND t1.c4 <= 20260417
AND t0.c2 = '10798'
AND t0.c3 = 0
AND t2.c2 >= 20260417
AND t2.c2 <= 99991231
),
z1 AS (
SELECT c0 FROM z0
UNION
SELECT c1 AS c0 FROM z0
)
SELECT p.c0, p.c1
FROM t3 p
JOIN z1 q ON q.c0 = p.c0
WHERE p.c2 = '10798';
```

plan of the original sql:
```
+-------------------------------------------+---------+-----------+--------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| id | estRows | task | access object | operator info |
+-------------------------------------------+---------+-----------+--------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| IndexHashJoin_53 | 2.50 | root | | inner join, inner:IndexLookUp_61, outer key:Column#42, inner key:test.t3.c0, equal cond:eq(Column#42, test.t3.c0) |
| ├─HashAgg_64(Build) | 2.00 | root | | group by:Column#42, funcs:firstrow(Column#42)->Column#42 |
| │ └─Union_65 | 2.00 | root | | |
| │ ├─StreamAgg_72 | 1.00 | root | | group by:test.t0.c0, funcs:firstrow(test.t0.c0)->test.t0.c0 |
| │ │ └─IndexHashJoin_156 | 1.56 | root | | inner join, inner:IndexLookUp_134, outer key:test.t1.c2, test.t1.c1, inner key:test.t2.c0, test.t2.c1, equal cond:eq(test.t1.c1, test.t2.c1), eq(test.t1.c2, test.t2.c0) |
| │ │ ├─IndexHashJoin_163(Build) | 1.25 | root | | inner join, inner:IndexLookUp_122, outer key:test.t0.c1, test.t0.c2, inner key:test.t1.c0, test.t1.c1, equal cond:eq(test.t0.c1, test.t1.c0), eq(test.t0.c2, test.t1.c1) |
| │ │ │ ├─Projection_168(Build) | 1.00 | root | | test.t0.c0, test.t0.c1, test.t0.c2, test.t0.c3 |
| │ │ │ │ └─IndexLookUp_167 | 1.00 | root | | |
| │ │ │ │ ├─IndexRangeScan_164(Build) | 10.00 | cop[tikv] | table:t0, index:idx_t0_c2_c0(c2, c0) | range:["10798","10798"], keep order:true, stats:pseudo |
| │ │ │ │ └─Selection_166(Probe) | 1.00 | cop[tikv] | | eq(test.t0.c3, 0) |
| │ │ │ │ └─TableRowIDScan_165 | 10.00 | cop[tikv] | table:t0 | keep order:false, stats:pseudo |
| │ │ │ └─IndexLookUp_122(Probe) | 1.25 | root | | |
| │ │ │ ├─Selection_120(Build) | 11.28 | cop[tikv] | | eq(test.t1.c1, "10798") |
| │ │ │ │ └─IndexRangeScan_118 | 3385.16 | cop[tikv] | table:t1, index:idx_t1_c0_c1(c0, c1) | range: decided by [eq(test.t1.c0, test.t0.c1) eq(test.t1.c1, test.t0.c2)], keep order:false, stats:pseudo |
| │ │ │ └─Selection_121(Probe) | 1.25 | cop[tikv] | | ge(test.t1.c3, 20260417), le(test.t1.c4, 20260417) |
| │ │ │ └─TableRowIDScan_119 | 11.28 | cop[tikv] | table:t1 | keep order:false, stats:pseudo |
| │ │ └─IndexLookUp_134(Probe) | 1.56 | root | | |
| │ │ ├─IndexRangeScan_131(Build) | 62.50 | cop[tikv] | table:t2, index:idx_t2_c0_c1(c0, c1) | range: decided by [eq(test.t2.c0, test.t1.c2) eq(test.t2.c1, test.t1.c1)], keep order:false, stats:pseudo |
| │ │ └─Selection_133(Probe) | 1.56 | cop[tikv] | | ge(test.t2.c2, 20260417), le(test.t2.c2, 99991231) |
| │ │ └─TableRowIDScan_132 | 62.50 | cop[tikv] | table:t2 | keep order:false, stats:pseudo |
| │ └─HashAgg_172 | 1.00 | root | | group by:test.t0.c1, funcs:firstrow(test.t0.c1)->test.t0.c1 |
| │ └─IndexHashJoin_181 | 1.56 | root | | inner join, inner:IndexLookUp_233, outer key:test.t1.c2, test.t1.c1, inner key:test.t2.c0, test.t2.c1, equal cond:eq(test.t1.c1, test.t2.c1), eq(test.t1.c2, test.t2.c0) |
| │ ├─IndexHashJoin_196(Build) | 1.25 | root | | inner join, inner:IndexLookUp_222, outer key:test.t0.c1, test.t0.c2, inner key:test.t1.c0, test.t1.c1, equal cond:eq(test.t0.c1, test.t1.c0), eq(test.t0.c2, test.t1.c1) |
| │ │ ├─IndexLookUp_217(Build) | 1.00 | root | | |
| │ │ │ ├─IndexRangeScan_214(Build) | 10.00 | cop[tikv] | table:t0, index:idx_t0_c2_c0(c2, c0) | range:["10798","10798"], keep order:false, stats:pseudo |
| │ │ │ └─Selection_216(Probe) | 1.00 | cop[tikv] | | eq(test.t0.c3, 0) |
| │ │ │ └─TableRowIDScan_215 | 10.00 | cop[tikv] | table:t0 | keep order:false, stats:pseudo |
| │ │ └─IndexLookUp_222(Probe) | 1.25 | root | | |
| │ │ ├─Selection_220(Build) | 11.28 | cop[tikv] | | eq(test.t1.c1, "10798") |
| │ │ │ └─IndexRangeScan_218 | 3385.16 | cop[tikv] | table:t1, index:idx_t1_c0_c1(c0, c1) | range: decided by [eq(test.t1.c0, test.t0.c1) eq(test.t1.c1, test.t0.c2)], keep order:false, stats:pseudo |
| │ │ └─Selection_221(Probe) | 1.25 | cop[tikv] | | ge(test.t1.c3, 20260417), le(test.t1.c4, 20260417) |
| │ │ └─TableRowIDScan_219 | 11.28 | cop[tikv] | table:t1 | keep order:false, stats:pseudo |
| │ └─IndexLookUp_233(Probe) | 1.56 | root | | |
| │ ├─IndexRangeScan_230(Build) | 62.50 | cop[tikv] | table:t2, index:idx_t2_c0_c1(c0, c1) | range: decided by [eq(test.t2.c0, test.t1.c2) eq(test.t2.c1, test.t1.c1)], keep order:false, stats:pseudo |
| │ └─Selection_232(Probe) | 1.56 | cop[tikv] | | ge(test.t2.c2, 20260417), le(test.t2.c2, 99991231) |
| │ └─TableRowIDScan_231 | 62.50 | cop[tikv] | table:t2 | keep order:false, stats:pseudo |
| └─IndexLookUp_61(Probe) | 2.50 | root | | |
| ├─IndexRangeScan_59(Build) | 2.50 | cop[tikv] | table:t3, index:idx_t3_c2_c0(c2, c0) | range: decided by [eq(test.t3.c0, Column#42) eq(test.t3.c2, 10798)], keep order:false, stats:pseudo |
| └─TableRowIDScan_60(Probe) | 2.50 | cop[tikv] | table:t3 | keep order:false, stats:pseudo |
+-------------------------------------------+---------+-----------+--------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
```

plan of the optimized SQL:
```
+----------------------------------+---------+-----------+--------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| id | estRows | task | access object | operator info |
+----------------------------------+---------+-----------+--------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| IndexHashJoin_129 | 2.50 | root | | inner join, inner:IndexLookUp_137, outer key:Column#34, inner key:test.t3.c0, equal cond:eq(Column#34, test.t3.c0) |
| ├─HashAgg_140(Build) | 2.00 | root | | group by:Column#34, funcs:firstrow(Column#34)->Column#34 |
| │ └─Union_141 | 2.50 | root | | |
| │ ├─Projection_143 | 1.25 | root | | test.t0.c0->Column#34 |
| │ │ └─Selection_145 | 1.25 | root | | not(isnull(test.t0.c0)) |
| │ │ └─CTEFullScan_146 | 1.56 | root | CTE:z0 | data:CTE_0 |
| │ └─Projection_147 | 1.25 | root | | test.t0.c1->Column#34 |
| │ └─Selection_149 | 1.25 | root | | not(isnull(test.t0.c1)) |
| │ └─CTEFullScan_150 | 1.56 | root | CTE:z0 | data:CTE_0 |
| └─IndexLookUp_137(Probe) | 2.50 | root | | |
| ├─IndexRangeScan_135(Build) | 2.50 | cop[tikv] | table:p, index:idx_t3_c2_c0(c2, c0) | range: decided by [eq(test.t3.c0, Column#34) eq(test.t3.c2, 10798)], keep order:false, stats:pseudo |
| └─TableRowIDScan_136(Probe) | 2.50 | cop[tikv] | table:p | keep order:false, stats:pseudo |
| CTE_0 | 1.56 | root | | Non-Recursive CTE |
| └─IndexHashJoin_45(Seed Part) | 1.56 | root | | inner join, inner:IndexLookUp_100, outer key:test.t1.c2, test.t1.c1, inner key:test.t2.c0, test.t2.c1, equal cond:eq(test.t1.c1, test.t2.c1), eq(test.t1.c2, test.t2.c0) |
| ├─IndexHashJoin_60(Build) | 1.25 | root | | inner join, inner:IndexLookUp_88, outer key:test.t0.c1, test.t0.c2, inner key:test.t1.c0, test.t1.c1, equal cond:eq(test.t0.c1, test.t1.c0), eq(test.t0.c2, test.t1.c1) |
| │ ├─IndexLookUp_83(Build) | 1.00 | root | | |
| │ │ ├─IndexRangeScan_80(Build) | 10.00 | cop[tikv] | table:t0, index:idx_t0_c2_c0(c2, c0) | range:["10798","10798"], keep order:false, stats:pseudo |
| │ │ └─Selection_82(Probe) | 1.00 | cop[tikv] | | eq(test.t0.c3, 0) |
| │ │ └─TableRowIDScan_81 | 10.00 | cop[tikv] | table:t0 | keep order:false, stats:pseudo |
| │ └─IndexLookUp_88(Probe) | 1.25 | root | | |
| │ ├─Selection_86(Build) | 11.28 | cop[tikv] | | eq(test.t1.c1, "10798") |
| │ │ └─IndexRangeScan_84 | 3385.16 | cop[tikv] | table:t1, index:idx_t1_c0_c1(c0, c1) | range: decided by [eq(test.t1.c0, test.t0.c1) eq(test.t1.c1, test.t0.c2)], keep order:false, stats:pseudo |
| │ └─Selection_87(Probe) | 1.25 | cop[tikv] | | ge(test.t1.c3, 20260417), le(test.t1.c4, 20260417) |
| │ └─TableRowIDScan_85 | 11.28 | cop[tikv] | table:t1 | keep order:false, stats:pseudo |
| └─IndexLookUp_100(Probe) | 1.56 | root | | |
| ├─IndexRangeScan_97(Build) | 62.50 | cop[tikv] | table:t2, index:idx_t2_c0_c1(c0, c1) | range: decided by [eq(test.t2.c0, test.t1.c2) eq(test.t2.c1, test.t1.c1)], keep order:false, stats:pseudo |
| └─Selection_99(Probe) | 1.56 | cop[tikv] | | ge(test.t2.c2, 20260417), le(test.t2.c2, 99991231) |
| └─TableRowIDScan_98 | 62.50 | cop[tikv] | table:t2 | keep order:false, stats:pseudo |
+----------------------------------+---------+-----------+--------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
```

Contributor guide

Open the contributing guide

Assessment

This issue has not been assessed yet.

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.