[X86][AVX512] fp16/bf16 compress/expand should lower to vpcompressw/vpexpandw instead of scalarization
- Dominant language
- LLVM
- Stars
- 40.5k
- Forks
- 18.7k
- PR merge metrics
- PR metrics pending
Description
https://godbolt.org/z/cvj17Kd3T
```llvm
define void @compressstore_v16f16_v16i1(ptr %base, <16 x half> %V, <16 x i1> %mask) {
call void @llvm.masked.compressstore.v16f16(<16 x half> %V, ptr %base, <16 x i1> %mask)
ret void
}
define void @compressstore_v16f16_v16i1_vpcompressw(ptr %base, <16 x i16> %V, <16 x i1> %mask) {
call void @llvm.masked.compressstore.v16f16(<16 x i16> %V, ptr %base, <16 x i1> %mask)
ret void
}
define void @compressstore_v16bf16_v16i1(ptr %base, <16 x bfloat> %V, <16 x i1> %mask) {
call void @llvm.masked.compressstore.v16bf16(<16 x bfloat> %V, ptr %base, <16 x i1> %mask)
ret void
}
define void @compressstore_v16bf16_v16i1_vpcompressw(ptr %base, <16 x i16> %V, <16 x i1> %mask) {
call void @llvm.masked.compressstore.v16bf16(<16 x i16> %V, ptr %base, <16 x i1> %mask)
ret void
}
```
```asm
compressstore_v16f16_v16i1:
sub rsp, 168
vmovups ymmword ptr [rsp + 112], ymm0
mov qword ptr [rsp + 144], rdi
vpsllw xmm0, xmm1, 7
vpmovb2m k0, xmm0
kmovd eax, k0
mov word ptr [rsp + 158], ax
and ax, 1
cmp ax, 0
mov qword ptr [rsp + 160], rdi
je .LBB0_2
mov rax, qword ptr [rsp + 144]
vmovups ymm0, ymmword ptr [rsp + 112]
vmovsh word ptr [rax], xmm0
add rax, 2
mov qword ptr [rsp + 160], rax
.LBB0_2:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp + 160]
mov qword ptr [rsp + 96], rax
and cx, 2
cmp cx, 0
mov qword ptr [rsp + 104], rax
je .LBB0_4
mov rax, qword ptr [rsp + 96]
vmovups ymm0, ymmword ptr [rsp + 112]
vpsrld xmm0, xmm0, 16
vmovsh word ptr [rax], xmm0
add rax, 2
mov qword ptr [rsp + 104], rax
.LBB0_4:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp + 104]
mov qword ptr [rsp + 80], rax
and cx, 4
cmp cx, 0
mov qword ptr [rsp + 88], rax
je .LBB0_6
mov rax, qword ptr [rsp + 80]
vmovups ymm0, ymmword ptr [rsp + 112]
vmovshdup xmm0, xmm0
vmovsh word ptr [rax], xmm0
add rax, 2
mov qword ptr [rsp + 88], rax
.LBB0_6:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp + 88]
mov qword ptr [rsp + 64], rax
and cx, 8
cmp cx, 0
mov qword ptr [rsp + 72], rax
je .LBB0_8
mov rax, qword ptr [rsp + 64]
vmovups ymm0, ymmword ptr [rsp + 112]
vpsrlq xmm0, xmm0, 48
vmovsh word ptr [rax], xmm0
add rax, 2
mov qword ptr [rsp + 72], rax
.LBB0_8:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp + 72]
mov qword ptr [rsp + 48], rax
and cx, 16
cmp cx, 0
mov qword ptr [rsp + 56], rax
je .LBB0_10
mov rax, qword ptr [rsp + 48]
vmovups ymm0, ymmword ptr [rsp + 112]
vpermilpd xmm0, xmm0, 1
vmovsh word ptr [rax], xmm0
add rax, 2
mov qword ptr [rsp + 56], rax
.LBB0_10:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp + 56]
mov qword ptr [rsp + 32], rax
and cx, 32
cmp cx, 0
mov qword ptr [rsp + 40], rax
je .LBB0_12
mov rax, qword ptr [rsp + 32]
vmovups ymm0, ymmword ptr [rsp + 112]
vpsrldq xmm0, xmm0, 10
vmovsh word ptr [rax], xmm0
add rax, 2
mov qword ptr [rsp + 40], rax
.LBB0_12:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp + 40]
mov qword ptr [rsp + 16], rax
and cx, 64
cmp cx, 0
mov qword ptr [rsp + 24], rax
je .LBB0_14
mov rax, qword ptr [rsp + 16]
vmovups ymm0, ymmword ptr [rsp + 112]
vpermilps xmm0, xmm0, 255
vmovsh word ptr [rax], xmm0
add rax, 2
mov qword ptr [rsp + 24], rax
.LBB0_14:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp + 24]
mov qword ptr [rsp], rax
and cx, 128
cmp cx, 0
mov qword ptr [rsp + 8], rax
je .LBB0_16
mov rax, qword ptr [rsp]
vmovups ymm0, ymmword ptr [rsp + 112]
vpsrldq xmm0, xmm0, 14
vmovsh word ptr [rax], xmm0
add rax, 2
mov qword ptr [rsp + 8], rax
.LBB0_16:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp + 8]
mov qword ptr [rsp - 16], rax
and cx, 256
cmp cx, 0
mov qword ptr [rsp - 8], rax
je .LBB0_18
mov rax, qword ptr [rsp - 16]
vmovups ymm0, ymmword ptr [rsp + 112]
vextractf128 xmm0, ymm0, 1
vmovsh word ptr [rax], xmm0
add rax, 2
mov qword ptr [rsp - 8], rax
.LBB0_18:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp - 8]
mov qword ptr [rsp - 32], rax
and cx, 512
cmp cx, 0
mov qword ptr [rsp - 24], rax
je .LBB0_20
mov rax, qword ptr [rsp - 32]
vmovups ymm0, ymmword ptr [rsp + 112]
vextractf128 xmm0, ymm0, 1
vpsrld xmm0, xmm0, 16
vmovsh word ptr [rax], xmm0
add rax, 2
mov qword ptr [rsp - 24], rax
.LBB0_20:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp - 24]
mov qword ptr [rsp - 48], rax
and cx, 1024
cmp cx, 0
mov qword ptr [rsp - 40], rax
je .LBB0_22
mov rax, qword ptr [rsp - 48]
vmovups ymm0, ymmword ptr [rsp + 112]
vextractf128 xmm0, ymm0, 1
vmovshdup xmm0, xmm0
vmovsh word ptr [rax], xmm0
add rax, 2
mov qword ptr [rsp - 40], rax
.LBB0_22:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp - 40]
mov qword ptr [rsp - 64], rax
and cx, 2048
cmp cx, 0
mov qword ptr [rsp - 56], rax
je .LBB0_24
mov rax, qword ptr [rsp - 64]
vmovups ymm0, ymmword ptr [rsp + 112]
vextractf128 xmm0, ymm0, 1
vpsrlq xmm0, xmm0, 48
vmovsh word ptr [rax], xmm0
add rax, 2
mov qword ptr [rsp - 56], rax
.LBB0_24:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp - 56]
mov qword ptr [rsp - 80], rax
and cx, 4096
cmp cx, 0
mov qword ptr [rsp - 72], rax
je .LBB0_26
mov rax, qword ptr [rsp - 80]
vmovups ymm0, ymmword ptr [rsp + 112]
vextractf128 xmm0, ymm0, 1
vpermilpd xmm0, xmm0, 1
vmovsh word ptr [rax], xmm0
add rax, 2
mov qword ptr [rsp - 72], rax
.LBB0_26:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp - 72]
mov qword ptr [rsp - 96], rax
and cx, 8192
cmp cx, 0
mov qword ptr [rsp - 88], rax
je .LBB0_28
mov rax, qword ptr [rsp - 96]
vmovups ymm0, ymmword ptr [rsp + 112]
vextractf128 xmm0, ymm0, 1
vpsrldq xmm0, xmm0, 10
vmovsh word ptr [rax], xmm0
add rax, 2
mov qword ptr [rsp - 88], rax
.LBB0_28:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp - 88]
mov qword ptr [rsp - 112], rax
and cx, 16384
cmp cx, 0
mov qword ptr [rsp - 104], rax
je .LBB0_30
mov rax, qword ptr [rsp - 112]
vmovups ymm0, ymmword ptr [rsp + 112]
vextractf128 xmm0, ymm0, 1
vpermilps xmm0, xmm0, 255
vmovsh word ptr [rax], xmm0
add rax, 2
mov qword ptr [rsp - 104], rax
.LBB0_30:
mov ax, word ptr [rsp + 158]
mov rcx, qword ptr [rsp - 104]
mov qword ptr [rsp - 120], rcx
and ax, -32768
cmp ax, 0
je .LBB0_32
mov rax, qword ptr [rsp - 120]
vmovups ymm0, ymmword ptr [rsp + 112]
vextractf128 xmm0, ymm0, 1
vpsrldq xmm0, xmm0, 14
vmovsh word ptr [rax], xmm0
.LBB0_32:
add rsp, 168
vzeroupper
ret
compressstore_v16f16_v16i1_vpcompressw:
vpsllw xmm1, xmm1, 7
vpmovb2m k1, xmm1
vpcompressw ymmword ptr [rdi] {k1}, ymm0
vzeroupper
ret
compressstore_v16bf16_v16i1:
sub rsp, 168
mov qword ptr [rsp + 104], rdi
vpsllw xmm1, xmm1, 7
vpmovb2m k0, xmm1
vmovups ymmword ptr [rsp + 112], ymm0
kmovd eax, k0
mov word ptr [rsp + 158], ax
and ax, 1
cmp ax, 0
mov qword ptr [rsp + 160], rdi
je .LBB2_2
mov rax, qword ptr [rsp + 104]
vmovups ymm0, ymmword ptr [rsp + 112]
vpextrw word ptr [rax], xmm0, 0
add rax, 2
mov qword ptr [rsp + 160], rax
.LBB2_2:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp + 160]
mov qword ptr [rsp + 88], rax
and cx, 2
cmp cx, 0
mov qword ptr [rsp + 96], rax
je .LBB2_4
mov rax, qword ptr [rsp + 88]
vmovups ymm0, ymmword ptr [rsp + 112]
vpextrw word ptr [rax], xmm0, 1
add rax, 2
mov qword ptr [rsp + 96], rax
.LBB2_4:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp + 96]
mov qword ptr [rsp + 72], rax
and cx, 4
cmp cx, 0
mov qword ptr [rsp + 80], rax
je .LBB2_6
mov rax, qword ptr [rsp + 72]
vmovups ymm0, ymmword ptr [rsp + 112]
vpextrw word ptr [rax], xmm0, 2
add rax, 2
mov qword ptr [rsp + 80], rax
.LBB2_6:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp + 80]
mov qword ptr [rsp + 56], rax
and cx, 8
cmp cx, 0
mov qword ptr [rsp + 64], rax
je .LBB2_8
mov rax, qword ptr [rsp + 56]
vmovups ymm0, ymmword ptr [rsp + 112]
vpextrw word ptr [rax], xmm0, 3
add rax, 2
mov qword ptr [rsp + 64], rax
.LBB2_8:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp + 64]
mov qword ptr [rsp + 40], rax
and cx, 16
cmp cx, 0
mov qword ptr [rsp + 48], rax
je .LBB2_10
mov rax, qword ptr [rsp + 40]
vmovups ymm0, ymmword ptr [rsp + 112]
vpextrw word ptr [rax], xmm0, 4
add rax, 2
mov qword ptr [rsp + 48], rax
.LBB2_10:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp + 48]
mov qword ptr [rsp + 24], rax
and cx, 32
cmp cx, 0
mov qword ptr [rsp + 32], rax
je .LBB2_12
mov rax, qword ptr [rsp + 24]
vmovups ymm0, ymmword ptr [rsp + 112]
vpextrw word ptr [rax], xmm0, 5
add rax, 2
mov qword ptr [rsp + 32], rax
.LBB2_12:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp + 32]
mov qword ptr [rsp + 8], rax
and cx, 64
cmp cx, 0
mov qword ptr [rsp + 16], rax
je .LBB2_14
mov rax, qword ptr [rsp + 8]
vmovups ymm0, ymmword ptr [rsp + 112]
vpextrw word ptr [rax], xmm0, 6
add rax, 2
mov qword ptr [rsp + 16], rax
.LBB2_14:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp + 16]
mov qword ptr [rsp - 8], rax
and cx, 128
cmp cx, 0
mov qword ptr [rsp], rax
je .LBB2_16
mov rax, qword ptr [rsp - 8]
vmovups ymm0, ymmword ptr [rsp + 112]
vpextrw word ptr [rax], xmm0, 7
add rax, 2
mov qword ptr [rsp], rax
.LBB2_16:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp]
mov qword ptr [rsp - 24], rax
and cx, 256
cmp cx, 0
mov qword ptr [rsp - 16], rax
je .LBB2_18
mov rax, qword ptr [rsp - 24]
vmovups ymm0, ymmword ptr [rsp + 112]
vextracti128 xmm0, ymm0, 1
vpextrw word ptr [rax], xmm0, 0
add rax, 2
mov qword ptr [rsp - 16], rax
.LBB2_18:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp - 16]
mov qword ptr [rsp - 40], rax
and cx, 512
cmp cx, 0
mov qword ptr [rsp - 32], rax
je .LBB2_20
mov rax, qword ptr [rsp - 40]
vmovups ymm0, ymmword ptr [rsp + 112]
vextracti128 xmm0, ymm0, 1
vpextrw word ptr [rax], xmm0, 1
add rax, 2
mov qword ptr [rsp - 32], rax
.LBB2_20:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp - 32]
mov qword ptr [rsp - 56], rax
and cx, 1024
cmp cx, 0
mov qword ptr [rsp - 48], rax
je .LBB2_22
mov rax, qword ptr [rsp - 56]
vmovups ymm0, ymmword ptr [rsp + 112]
vextracti128 xmm0, ymm0, 1
vpextrw word ptr [rax], xmm0, 2
add rax, 2
mov qword ptr [rsp - 48], rax
.LBB2_22:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp - 48]
mov qword ptr [rsp - 72], rax
and cx, 2048
cmp cx, 0
mov qword ptr [rsp - 64], rax
je .LBB2_24
mov rax, qword ptr [rsp - 72]
vmovups ymm0, ymmword ptr [rsp + 112]
vextracti128 xmm0, ymm0, 1
vpextrw word ptr [rax], xmm0, 3
add rax, 2
mov qword ptr [rsp - 64], rax
.LBB2_24:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp - 64]
mov qword ptr [rsp - 88], rax
and cx, 4096
cmp cx, 0
mov qword ptr [rsp - 80], rax
je .LBB2_26
mov rax, qword ptr [rsp - 88]
vmovups ymm0, ymmword ptr [rsp + 112]
vextracti128 xmm0, ymm0, 1
vpextrw word ptr [rax], xmm0, 4
add rax, 2
mov qword ptr [rsp - 80], rax
.LBB2_26:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp - 80]
mov qword ptr [rsp - 104], rax
and cx, 8192
cmp cx, 0
mov qword ptr [rsp - 96], rax
je .LBB2_28
mov rax, qword ptr [rsp - 104]
vmovups ymm0, ymmword ptr [rsp + 112]
vextracti128 xmm0, ymm0, 1
vpextrw word ptr [rax], xmm0, 5
add rax, 2
mov qword ptr [rsp - 96], rax
.LBB2_28:
mov cx, word ptr [rsp + 158]
mov rax, qword ptr [rsp - 96]
mov qword ptr [rsp - 120], rax
and cx, 16384
cmp cx, 0
mov qword ptr [rsp - 112], rax
je .LBB2_30
mov rax, qword ptr [rsp - 120]
vmovups ymm0, ymmword ptr [rsp + 112]
vextracti128 xmm0, ymm0, 1
vpextrw word ptr [rax], xmm0, 6
add rax, 2
mov qword ptr [rsp - 112], rax
.LBB2_30:
mov ax, word ptr [rsp + 158]
mov rcx, qword ptr [rsp - 112]
mov qword ptr [rsp - 128], rcx
and ax, -32768
cmp ax, 0
je .LBB2_32
mov rax, qword ptr [rsp - 128]
vmovups ymm0, ymmword ptr [rsp + 112]
vextracti128 xmm0, ymm0, 1
vpextrw word ptr [rax], xmm0, 7
.LBB2_32:
add rsp, 168
vzeroupper
ret
compressstore_v16bf16_v16i1_vpcompressw:
vpsllw xmm1, xmm1, 7
vpmovb2m k1, xmm1
vpcompressw ymmword ptr [rdi] {k1}, ymm0
vzeroupper
ret
```
Contributor guide
Assessment
This issue has not been assessed yet.