llvm / llvm/llvm-project

[X86][AVX512] failed to fold non-pow2 masked load/store to equivalent pow2 unmasked load/store

Open
#205,032 1 comment 0 reactions 0 assignees View on GitHub
backend:X86 llvm:SelectionDAG missed-optimization
Dominant language
LLVM
Stars
40.5k
Forks
18.7k
PR merge metrics
PR metrics pending

Description

https://godbolt.org/z/oMxodv1dr

```llvm
define void @load_store_v4i64(ptr %0, ptr %1) {
%3 = load <4 x i64>, ptr %0
store <4 x i64> %3, ptr %1
ret void
}

define void @load_store_masked_v5i64(ptr %0, ptr %1) {
%3 = tail call <5 x i64> @llvm.masked.load.v5i64.p0(ptr %0, <5 x i1> , <5 x i64> poison)
tail call void @llvm.masked.store.v5i64.p0(<5 x i64> %3 , ptr %1, <5 x i1> )
ret void
}

define void @load_store_v8i32(ptr %0, ptr %1) {
%3 = load <8 x i32>, ptr %0
store <8 x i32> %3, ptr %1
ret void
}

define void @load_store_masked_v9i32(ptr %0, ptr %1) {
%3 = tail call <9 x i32> @llvm.masked.load.v9i32.p0(ptr %0, <9 x i1> , <9 x i32> poison)
tail call void @llvm.masked.store.v9i32.p0(<9 x i32> %3 , ptr %1, <9 x i1> )
ret void
}

define void @load_store_v16i8(ptr %0, ptr %1) {
%3 = load <16 x i8>, ptr %0
store <16 x i8> %3, ptr %1
ret void
}

define void @load_store_v17i8(ptr %0, ptr %1) {
%3 = tail call <17 x i8> @llvm.masked.load.v17i8.p0(ptr %0, <17 x i1> , <17 x i8> poison)
tail call void @llvm.masked.store.v17i8.p0(<17 x i8> %3 , ptr %1, <17 x i1> )
ret void
}

define void @load_store_v32i8(ptr %0, ptr %1) {
%3 = load <32 x i8>, ptr %0
store <32 x i8> %3, ptr %1
ret void
}

define void @load_store_v33i8(ptr %0, ptr %1) {
%3 = tail call <33 x i8> @llvm.masked.load.v33i8.p0(ptr %0, <33 x i1> , <33 x i8> poison)
tail call void @llvm.masked.store.v33i8.p0(<33 x i8> %3 , ptr %1, <33 x i1> )
ret void
}
```
```asm
load_store_v4i64: # @load_store_v4i64
vmovaps ymm0, ymmword ptr [rdi]
vmovaps ymmword ptr [rsi], ymm0
vzeroupper
ret
load_store_masked_v5i64: # @load_store_masked_v5i64
mov al, 15
kmovd k1, eax
vmovdqu64 zmm0 {k1} {z}, zmmword ptr [rdi]
vmovdqu64 zmmword ptr [rsi] {k1}, zmm0
vzeroupper
ret
load_store_v8i32: # @load_store_v8i32
vmovaps ymm0, ymmword ptr [rdi]
vmovaps ymmword ptr [rsi], ymm0
vzeroupper
ret
load_store_masked_v9i32: # @load_store_masked_v9i32
kxnorb k1, k0, k0
vmovdqu32 zmm0 {k1} {z}, zmmword ptr [rdi]
vmovdqu32 zmmword ptr [rsi] {k1}, zmm0
vzeroupper
ret
load_store_v16i8: # @load_store_v16i8
vmovaps xmm0, xmmword ptr [rdi]
vmovaps xmmword ptr [rsi], xmm0
ret
load_store_v17i8: # @load_store_v17i8
kxnorw k1, k0, k0
vmovdqu8 ymm0 {k1} {z}, ymmword ptr [rdi]
vmovdqu8 ymmword ptr [rsi] {k1}, ymm0
vzeroupper
ret
load_store_v32i8: # @load_store_v32i8
vmovaps ymm0, ymmword ptr [rdi]
vmovaps ymmword ptr [rsi], ymm0
vzeroupper
ret
load_store_v33i8: # @load_store_v33i8
kxnord k1, k0, k0
vmovdqu8 zmm0 {k1} {z}, zmmword ptr [rdi]
vmovdqu8 zmmword ptr [rsi] {k1}, zmm0
vzeroupper
ret
```

But if the equivalent pow2 unmasked load/store happened to fill zmm, it can be correctly folded.

```llvm
define void @load_store_v16i8(ptr %0, ptr %1) {
%3 = load <16 x i32>, ptr %0
store <16 x i32> %3, ptr %1
ret void
}

define void @load_store_v17i8(ptr %0, ptr %1) {
%3 = tail call <17 x i32> @llvm.masked.load.v17i32.p0(ptr %0, <17 x i1> , <17 x i32> poison)
tail call void @llvm.masked.store.v17i32.p0(<17 x i32> %3 , ptr %1, <17 x i1> )
ret void
}

define void @load_store_v8i64(ptr %0, ptr %1) {
%3 = load <8 x i64>, ptr %0
store <8 x i64> %3, ptr %1
ret void
}

define void @load_store_masked_v9i64(ptr %0, ptr %1) {
%3 = tail call <9 x i64> @llvm.masked.load.v9i64.p0(ptr %0, <9 x i1> , <9 x i64> poison)
tail call void @llvm.masked.store.v9i64.p0(<9 x i64> %3 , ptr %1, <9 x i1> )
ret void
}
```
```asm
load_store_v16i8: # @load_store_v16i8
vmovaps zmm0, zmmword ptr [rdi]
vmovaps zmmword ptr [rsi], zmm0
vzeroupper
ret
load_store_v17i8: # @load_store_v17i8
vmovups zmm0, zmmword ptr [rdi]
vmovups zmmword ptr [rsi], zmm0
vzeroupper
ret
load_store_v8i64: # @load_store_v8i64
vmovaps zmm0, zmmword ptr [rdi]
vmovaps zmmword ptr [rsi], zmm0
vzeroupper
ret
load_store_masked_v9i64: # @load_store_masked_v9i64
vmovups zmm0, zmmword ptr [rdi]
vmovups zmmword ptr [rsi], zmm0
vzeroupper
ret
```

Contributor guide

Open the contributing guide

Research direction

Use the linked Godbolt example as the entry point and reproduce the listed LLVM IR and x86 AVX512 assembly for the masked and unmasked vector cases. Compare the non-power-of-two results with the cases that already fold to zmm operations; done means the safe equivalent masked load/store cases produce the corresponding power-of-two unmasked instructions.

Written by the indexing model from the issue text.

Assessment

Domain
compilers, performance
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Quiet
Clarity
Mostly clear
Newbie friendliness
45/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.