llvm / llvm/llvm-project

SLP Vectorizer creates code much slower than scalar version when AVX512 is enabled.

Open
#223,995 1 comment 0 reactions 0 assignees View on GitHub
backend:X86 llvm:SLPVectorizer performance
Dominant language
LLVM
Stars
40.5k
Forks
18.7k
PR merge metrics
PR metrics pending

Description

The following IR will get vectorized by Sif the target cpu has AVX512 festures:
```llvm
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"

define void @circular_index_update(ptr noalias %state, ptr noalias %seed, i32 %size) {
entry:
%i_ptr = getelementptr inbounds i32, ptr %state, i64 0
%j_ptr = getelementptr inbounds i32, ptr %state, i64 1

%i = load i32, ptr %i_ptr
%j = load i32, ptr %j_ptr
%size_minus_1 = add i32 %size, -1

%i_is_zero = icmp eq i32 %i, 0
%i_minus_1 = add i32 %i, -1
%i_next = select i1 %i_is_zero, i32 %size_minus_1, i32 %i_minus_1
%i_next_z = zext i32 %i_next to i64
%seed_i_ptr = getelementptr inbounds i32, ptr %seed, i64 %i_next_z
%seed_i = load i32, ptr %seed_i_ptr
store i32 %i_next, ptr %i_ptr

%j_is_zero = icmp eq i32 %j, 0
%j_minus_1 = add i32 %j, -1
%j_next = select i1 %j_is_zero, i32 %size_minus_1, i32 %j_minus_1
%j_next_z = zext i32 %j_next to i64
%seed_j_ptr = getelementptr inbounds i32, ptr %seed, i64 %j_next_z
%seed_j = load i32, ptr %seed_j_ptr
store i32 %j_next, ptr %j_ptr

%diff = sub i32 %seed_i, %seed_j
store i32 %diff, ptr %seed_i_ptr
ret void
}
```
to
```llvm
define void @circular_index_update(ptr noalias %state, ptr noalias %seed, i32 %size) #0 {
entry:
%i_ptr = getelementptr inbounds i32, ptr %state, i64 0
%size_minus_1 = add i32 %size, -1
%0 = load <2 x i32>, ptr %i_ptr, align 4
%1 = icmp eq <2 x i32> %0, zeroinitializer
%2 = add <2 x i32> %0, splat (i32 -1)
%3 = insertelement <2 x i32> poison, i32 %size_minus_1, i64 0
%4 = shufflevector <2 x i32> %3, <2 x i32> poison, <2 x i32> zeroinitializer
%5 = select <2 x i1> %1, <2 x i32> %4, <2 x i32> %2
%6 = extractelement <2 x i32> %5, i64 0
%i_next_z = zext i32 %6 to i64
%seed_i_ptr = getelementptr inbounds i32, ptr %seed, i64 %i_next_z
%seed_i = load i32, ptr %seed_i_ptr, align 4
%7 = extractelement <2 x i32> %5, i64 1
%j_next_z = zext i32 %7 to i64
%seed_j_ptr = getelementptr inbounds i32, ptr %seed, i64 %j_next_z
%seed_j = load i32, ptr %seed_j_ptr, align 4
store <2 x i32> %5, ptr %i_ptr, align 4
%diff = sub i32 %seed_i, %seed_j
store i32 %diff, ptr %seed_i_ptr, align 4
ret void
}
```
See https://godbolt.org/z/4ndEYe1cT

This vectorized version produces significantly slower asm, as verified by `llvm-mca` and real runs on a CPU with rocket lake architecture:
vectorized: https://godbolt.org/z/rPh4YM8ce
```
Iterations: 100
Instructions: 1300
Total Cycles: 1311
Total uOps: 1800

Dispatch Width: 6
uOps Per Cycle: 1.37
IPC: 0.99
Block RThroughput: 3.0

Instruction Info:
[1]: #uOps
[2]: Latency
[3]: RThroughput
[4]: MayLoad
[5]: MayStore
[6]: HasSideEffects (U)

[1] [2] [3] [4] [5] [6] Instructions:
1 1 0.25 dec edx
1 5 0.50 * vmovq xmm0, qword ptr [rdi]
1 4 1.00 vptestmd k1, xmm0, xmm0
1 1 0.50 vpcmpeqd xmm1, xmm1, xmm1
1 1 1.00 vpbroadcastd xmm2, edx
1 1 0.33 vpaddd xmm2 {k1}, xmm0, xmm1
1 2 1.00 vmovd eax, xmm2
2 3 1.00 vpextrd ecx, xmm2, 1
1 5 0.50 * mov edx, dword ptr [rsi + 4*rax]
2 6 0.50 * sub edx, dword ptr [rsi + 4*rcx]
2 1 0.50 * vmovq qword ptr [rdi], xmm2
1 1 0.50 * mov dword ptr [rsi + 4*rax], edx
3 7 1.00 U ret

Resources:
[0] - ICXDivider
[1] - ICXFPDivider
[2] - ICXPort0
[3] - ICXPort1
[4] - ICXPort2
[5] - ICXPort3
[6] - ICXPort4
[7] - ICXPort5
[8] - ICXPort6
[9] - ICXPort7
[10] - ICXPort8
[11] - ICXPort9

Resource pressure per iteration:
[0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11]
- - 3.00 2.98 2.00 2.00 1.00 2.99 2.03 1.00 1.00 1.00

Resource pressure by instruction:
[0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11] Instructions:
- - 0.02 0.01 - - - 0.01 0.96 - - - dec edx
- - - - 0.94 0.06 - - - - - - vmovq xmm0, qword ptr [rdi]
- - - - - - - 1.00 - - - - vptestmd k1, xmm0, xmm0
- - 0.02 0.98 - - - - - - - - vpcmpeqd xmm1, xmm1, xmm1
- - - - - - - 1.00 - - - - vpbroadcastd xmm2, edx
- - - 0.99 - - - 0.01 - - - - vpaddd xmm2 {k1}, xmm0, xmm1
- - 1.00 - - - - - - - - - vmovd eax, xmm2
- - 1.00 0.96 - - - 0.04 - - - - vpextrd ecx, xmm2, 1
- - - - 0.01 0.99 - - - - - - mov edx, dword ptr [rsi + 4*rax]
- - 0.91 0.02 0.07 0.93 - - 0.07 - - - sub edx, dword ptr [rsi + 4*rcx]
- - - - - - - - - - 1.00 1.00 vmovq qword ptr [rdi], xmm2
- - - - - - 1.00 - - 1.00 - - mov dword ptr [rsi + 4*rax], edx
- - 0.05 0.02 0.98 0.02 - 0.93 1.00 - - - ret
```
vs scalar: https://godbolt.org/z/rPh4YM8ce
```
Iterations: 100
Instructions: 1300
Total Cycles: 313
Total uOps: 1600

Dispatch Width: 6
uOps Per Cycle: 5.11
IPC: 4.15
Block RThroughput: 2.7

Instruction Info:
[1]: #uOps
[2]: Latency
[3]: RThroughput
[4]: MayLoad
[5]: MayStore
[6]: HasSideEffects (U)

[1] [2] [3] [4] [5] [6] Instructions:
1 5 0.50 * mov eax, dword ptr [rdi]
1 5 0.50 * mov ecx, dword ptr [rdi + 4]
1 1 0.25 dec edx
1 1 0.25 sub eax, 1
1 1 0.50 cmovb eax, edx
1 5 0.50 * mov r8d, dword ptr [rsi + 4*rax]
1 1 0.50 * mov dword ptr [rdi], eax
1 1 0.25 sub ecx, 1
1 1 0.50 cmovb ecx, edx
2 6 0.50 * sub r8d, dword ptr [rsi + 4*rcx]
1 1 0.50 * mov dword ptr [rdi + 4], ecx
1 1 0.50 * mov dword ptr [rsi + 4*rax], r8d
3 7 1.00 U ret

Resources:
[0] - ICXDivider
[1] - ICXFPDivider
[2] - ICXPort0
[3] - ICXPort1
[4] - ICXPort2
[5] - ICXPort3
[6] - ICXPort4
[7] - ICXPort5
[8] - ICXPort6
[9] - ICXPort7
[10] - ICXPort8
[11] - ICXPort9

Resource pressure per iteration:
[0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11]
- - 2.00 1.99 2.50 2.50 1.50 2.00 2.01 1.50 1.50 1.50

Resource pressure by instruction:
[0] [1] [2] [3] [4] [5] [6] [7] [8] [9] [10] [11] Instructions:
- - - - 0.49 0.51 - - - - - - mov eax, dword ptr [rdi]
- - - - 0.51 0.49 - - - - - - mov ecx, dword ptr [rdi + 4]
- - - 0.98 - - - 0.01 0.01 - - - dec edx
- - 0.04 0.95 - - - 0.01 - - - - sub eax, 1
- - 0.01 - - - - - 0.99 - - - cmovb eax, edx
- - - - 0.49 0.51 - - - - - - mov r8d, dword ptr [rsi + 4*rax]
- - - - - - 0.50 - - 0.50 0.50 0.50 mov dword ptr [rdi], eax
- - 0.95 0.01 - - - 0.04 - - - - sub ecx, 1
- - 0.99 - - - - - 0.01 - - - cmovb ecx, edx
- - - 0.04 0.51 0.49 - 0.96 - - - - sub r8d, dword ptr [rsi + 4*rcx]
- - - - - - 0.50 - - 0.50 0.50 0.50 mov dword ptr [rdi + 4], ecx
- - - - - - 0.50 - - 0.50 0.50 0.50 mov dword ptr [rsi + 4*rax], r8d
- - 0.01 0.01 0.50 0.50 - 0.98 1.00 - - - ret

```

LP

Contributor guide

Open the contributing guide

Research direction

Start with the SLP Vectorizer using the LLVM IR reproducer in the issue, then compare the generated code against the scalar baseline linked from Compiler Explorer. Use llvm-mca and a Rocket Lake system to reproduce the throughput difference; done means the AVX512-enabled path no longer regresses for this case and the comparison remains validated.

Written by the indexing model from the issue text.

Assessment

Domain
compilers, performance
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Active
Clarity
Mostly clear
Newbie friendliness
48/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.