[AArch64][SVE] failed to use sve2 vector s/umulh
- Dominant language
- LLVM
- Stars
- 40.5k
- Forks
- 18.7k
- PR merge metrics
- PR metrics pending
Description
found from https://github.com/gcc-mirror/gcc/commit/75ee89676c8c61263b21c1bbf90ad4fd42c417c1
https://godbolt.org/z/9zE4Th7PK
```c
void
mulh_s64 (int64_t *restrict dst, int64_t *restrict a, int64_t *restrict b,
int count)
{
for (int i = 0; i < count; ++i)
dst[i] = (int64_t) (((__int128) a[i] * b[i]) >> 64);
}
void
mulh_u64 (uint64_t *restrict dst, uint64_t *restrict a, uint64_t *restrict b,
int count)
{
for (int i = 0; i < count; ++i)
dst[i] = (uint64_t) (((unsigned __int128) a[i] * b[i]) >> 64);
}
```
gcc:
```asm
mulh_s64:
cble w3, wzr, .L1
mov w4, 0
whilelo p7.d, wzr, w3
.L3:
ld1d z31.d, p7/z, [x1, x4, lsl 3]
ld1d z30.d, p7/z, [x2, x4, lsl 3]
smulh z30.d, z31.d, z30.d
st1d z30.d, p7, [x0, x4, lsl 3]
incd x4
whilelo p7.d, w4, w3
b.any .L3
.L1:
ret
mulh_u64:
cble w3, wzr, .L6
mov w4, 0
whilelo p7.d, wzr, w3
.L8:
ld1d z31.d, p7/z, [x1, x4, lsl 3]
ld1d z30.d, p7/z, [x2, x4, lsl 3]
umulh z30.d, z31.d, z30.d
st1d z30.d, p7, [x0, x4, lsl 3]
incd x4
whilelo p7.d, w4, w3
b.any .L8
.L6:
ret
```
clang:
```asm
mulh_s64:
cblt w3, #1, .LBB0_8
mov w8, w3
cbhi w3, #3, .LBB0_3
mov x9, xzr
b .LBB0_6
.LBB0_3:
and x9, x8, #0x7ffffffc
add x10, x2, #16
add x11, x0, #16
add x12, x1, #16
and x13, x8, #0x7ffffffc
.LBB0_4:
ldp x14, x17, [x10, #-16]
subs x13, x13, #4
ldp x15, x16, [x12, #-16]
smulh x14, x14, x15
ldp x15, x3, [x10], #32
smulh x16, x17, x16
ldp x17, x18, [x12], #32
smulh x15, x15, x17
stp x14, x16, [x11, #-16]
smulh x17, x3, x18
stp x15, x17, [x11], #32
b.ne .LBB0_4
cbeq x9, x8, .LBB0_8
.LBB0_6:
lsl x12, x9, #3
sub x8, x8, x9
add x10, x0, x12
add x11, x2, x12
add x12, x1, x12
.LBB0_7:
ldr x9, [x12], #8
subs x8, x8, #1
ldr x13, [x11], #8
smulh x9, x13, x9
str x9, [x10], #8
b.ne .LBB0_7
.LBB0_8:
ret
mulh_u64:
cblt w3, #1, .LBB1_8
mov w8, w3
cbhi w3, #3, .LBB1_3
mov x9, xzr
b .LBB1_6
.LBB1_3:
and x9, x8, #0x7ffffffc
add x10, x2, #16
add x11, x0, #16
add x12, x1, #16
and x13, x8, #0x7ffffffc
.LBB1_4:
ldp x14, x17, [x10, #-16]
subs x13, x13, #4
ldp x15, x16, [x12, #-16]
umulh x14, x14, x15
ldp x15, x3, [x10], #32
umulh x16, x17, x16
ldp x17, x18, [x12], #32
umulh x15, x15, x17
stp x14, x16, [x11, #-16]
umulh x17, x3, x18
stp x15, x17, [x11], #32
b.ne .LBB1_4
cbeq x9, x8, .LBB1_8
.LBB1_6:
lsl x12, x9, #3
sub x8, x8, x9
add x10, x0, x12
add x11, x2, x12
add x12, x1, x12
.LBB1_7:
ldr x9, [x12], #8
subs x8, x8, #1
ldr x13, [x11], #8
umulh x9, x13, x9
str x9, [x10], #8
b.ne .LBB1_7
.LBB1_8:
ret
```
For reference, x86 can vectorize this loop, so this may not be a middle-end issue:
```asm
.LCPI0_0:
.quad 4294967295
mulh_s64:
test ecx, ecx
jle .LBB0_7
mov r8, rdx
mov r9d, ecx
cmp ecx, 8
jae .LBB0_3
xor ecx, ecx
jmp .LBB0_6
.LBB0_3:
{nf} and ecx, r9d, 2147483640
{nf} shr eax, r9d, 3
and eax, 268435455
shl rax, 6
xor edx, edx
vpbroadcastq zmm0, qword ptr [rip + .LCPI0_0]
.LBB0_4:
vmovdqu64 zmm1, zmmword ptr [rsi + rdx]
vmovdqu64 zmm2, zmmword ptr [r8 + rdx]
vpandq zmm3, zmm1, zmm0
vpsraq zmm4, zmm2, 32
vpmullq zmm3, zmm4, zmm3
vpmuludq zmm5, zmm2, zmm1
vpsrlq zmm5, zmm5, 32
vpaddq zmm3, zmm3, zmm5
vpsraq zmm5, zmm3, 32
vpandq zmm3, zmm3, zmm0
vpandq zmm2, zmm2, zmm0
vpsraq zmm1, zmm1, 32
vpmullq zmm2, zmm2, zmm1
vpaddq zmm2, zmm2, zmm3
vpsraq zmm2, zmm2, 32
vpmuldq zmm1, zmm4, zmm1
vpaddq zmm1, zmm1, zmm5
vpaddq zmm1, zmm1, zmm2
vmovdqu64 zmmword ptr [rdi + rdx], zmm1
add rdx, 64
cmp rax, rdx
jne .LBB0_4
cmp ecx, r9d
je .LBB0_7
.LBB0_6:
mov rax, qword ptr [r8 + 8*rcx]
imul qword ptr [rsi + 8*rcx]
mov qword ptr [rdi + 8*rcx], rdx
inc rcx
cmp r9, rcx
jne .LBB0_6
.LBB0_7:
vzeroupper
ret
.LCPI1_0:
.quad 4294967295
mulh_u64:
test ecx, ecx
jle .LBB1_7
mov rax, rdx
mov r8d, ecx
cmp ecx, 8
jae .LBB1_3
xor ecx, ecx
jmp .LBB1_6
.LBB1_3:
{nf} and ecx, r8d, 2147483640
{nf} shr edx, r8d, 3
and edx, 268435455
shl rdx, 6
xor r9d, r9d
vpbroadcastq zmm0, qword ptr [rip + .LCPI1_0]
.LBB1_4:
vmovdqu64 zmm1, zmmword ptr [rsi + r9]
vmovdqu64 zmm2, zmmword ptr [rax + r9]
vpsrlq zmm3, zmm2, 32
vpmuludq zmm4, zmm3, zmm1
vpmuludq zmm5, zmm2, zmm1
vpsrlq zmm5, zmm5, 32
vpaddq zmm4, zmm4, zmm5
vpsrlq zmm5, zmm4, 32
vpandq zmm4, zmm4, zmm0
vpsrlq zmm1, zmm1, 32
vpmuludq zmm2, zmm2, zmm1
vpaddq zmm2, zmm2, zmm4
vpsrlq zmm2, zmm2, 32
vpmuludq zmm1, zmm3, zmm1
vpaddq zmm1, zmm1, zmm5
vpaddq zmm1, zmm1, zmm2
vmovdqu64 zmmword ptr [rdi + r9], zmm1
add r9, 64
cmp rdx, r9
jne .LBB1_4
cmp ecx, r8d
je .LBB1_7
.LBB1_6:
mov rdx, qword ptr [rax + 8*rcx]
mulx rdx, rdx, qword ptr [rsi + 8*rcx]
mov qword ptr [rdi + 8*rcx], rdx
inc rcx
cmp r8, rcx
jne .LBB1_6
.LBB1_7:
vzeroupper
ret
```
But mulh can indeed be created from i64 vector constant division:
```c
#include
typedef uint64_t u64 [[gnu::vector_size(16)]];
typedef int64_t s64 [[gnu::vector_size(16)]];
u64 foo(u64 a){
return a / 7;
}
s64 bar(s64 a){
return a / 7;
}
```
```asm
foo:
mov x8, #9363
movk x8, #37449, lsl #16
movk x8, #18724, lsl #32
movk x8, #9362, lsl #48
mov z1.d, x8
umulh z1.d, z0.d, z1.d
sub v0.2d, v0.2d, v1.2d
usra v1.2d, v0.2d, #1
ushr v0.2d, v1.2d, #2
ret
bar:
mov x8, #18725
movk x8, #9362, lsl #16
movk x8, #37449, lsl #32
movk x8, #18724, lsl #48
mov z1.d, x8
smulh z1.d, z0.d, z1.d
sshr v0.2d, v1.2d, #1
usra v0.2d, v1.2d, #63
ret
```
Contributor guide
Research direction
Reproduce the two C loops with AArch64 SVE enabled and compare the GCC and Clang assembly shown in the issue. Use the vector-constant division examples as a reference for existing smulh and umulh generation. Done means the SVE2 loops select the appropriate signed or unsigned high-multiply instructions while preserving the shown results.
Written by the indexing model from the issue text.
Assessment
- Tech stack
- c
- Domain
- compilers, performance
- Issue type
- Bug
- Difficulty
- 4/5
- Estimated time
- 3-5 days
- Activity status
- Active
- Clarity
- Mostly clear
- Newbie friendliness
- 48/100