rust-lang / rust-lang/rust

Several bugs stemming from use of `f32`/`float` for discarded vector outputs in x86-64 inline asm

Open
#159,418 4 comments 0 reactions 0 assignees View on GitHub

Nobody has claimed this yet.

A-LLVM C-bug needs-triage O-x86_64
Dominant language
Rust
Stars
119k
Forks
16.1k
PR metrics pending

Description

out(zmm_reg) _ is currently treated differently than out(zmm_reg) some_var (more details in later comment), resulting in the following bugs:

1: Missing vzeroupper

2&3: Despite AVX512 having 32 registers (SSE and AVX2 had 16), inline asm sometimes fails when allocating a scratch register - out(zmm_reg) _ - after ~16 registers are in use. The type of error depends on whether avx512vl is present. Explained in later comment

Original reproducer

Here is a small reproducer (compile in release mode):

use std::arch::x86_64::*;

#[target_feature(enable = "avx512f,avx512bw,avx512vl")]
pub fn p0(input: &[__m512i]) -> usize {
    let one_v = _mm512_set1_epi8(1 as i8);
    let b0_v = _mm512_set1_epi8(b'0' as i8);
    let b1_v = _mm512_set1_epi8(b'1' as i8);
    let b2_v = _mm512_set1_epi8(b'2' as i8);
    let b3_v = _mm512_set1_epi8(b'3' as i8);
    let b4_v = _mm512_set1_epi8(b'4' as i8);
    let b5_v = _mm512_set1_epi8(b'5' as i8);
    let b6_v = _mm512_set1_epi8(b'6' as i8);
    let b7_v = _mm512_set1_epi8(b'7' as i8);
    let b8_v = _mm512_set1_epi8(b'8' as i8);
    let b9_v = _mm512_set1_epi8(b'9' as i8);
    let b10_v = _mm512_set1_epi8(b'x' as i8);
    let b11_v = _mm512_set1_epi8(b'y' as i8);
    let b12_v = _mm512_set1_epi8(b'z' as i8);
    let b13_v = _mm512_set1_epi8(b'a' as i8);
    let b14_v = _mm512_set1_epi8(b'b' as i8);

    let mut acc = _mm512_setzero_si512();
    for &v in input {
        acc = _mm512_mask_add_epi8(acc, _mm512_cmpeq_epi8_mask(v, b0_v), acc, one_v);
        acc = _mm512_mask_add_epi8(acc, _mm512_cmpeq_epi8_mask(v, b1_v), acc, one_v);
        acc = _mm512_mask_add_epi8(acc, _mm512_cmpeq_epi8_mask(v, b2_v), acc, one_v);
        acc = _mm512_mask_add_epi8(acc, _mm512_cmpeq_epi8_mask(v, b3_v), acc, one_v);
        acc = _mm512_mask_add_epi8(acc, _mm512_cmpeq_epi8_mask(v, b4_v), acc, one_v);
        acc = _mm512_mask_add_epi8(acc, _mm512_cmpeq_epi8_mask(v, b5_v), acc, one_v);
        acc = _mm512_mask_add_epi8(acc, _mm512_cmpeq_epi8_mask(v, b6_v), acc, one_v);
        acc = _mm512_mask_add_epi8(acc, _mm512_cmpeq_epi8_mask(v, b7_v), acc, one_v);
        acc = _mm512_mask_add_epi8(acc, _mm512_cmpeq_epi8_mask(v, b8_v), acc, one_v);
        acc = _mm512_mask_add_epi8(acc, _mm512_cmpeq_epi8_mask(v, b9_v), acc, one_v);
        acc = _mm512_mask_add_epi8(acc, _mm512_cmpeq_epi8_mask(v, b10_v), acc, one_v);
        acc = _mm512_mask_add_epi8(acc, _mm512_cmpeq_epi8_mask(v, b11_v), acc, one_v);
        acc = _mm512_mask_add_epi8(acc, _mm512_cmpeq_epi8_mask(v, b12_v), acc, one_v);
        acc = _mm512_mask_add_epi8(acc, _mm512_cmpeq_epi8_mask(v, b13_v), acc, one_v);
        acc = _mm512_mask_add_epi8(acc, _mm512_cmpeq_epi8_mask(v, b14_v), acc, one_v);
        unsafe {
            std::arch::asm!(
                "vpaddb {tmp}, {out}, {out}",
                "vpaddb {out}, {tmp}, {tmp}",
                "vpaddb {out}, {out}, {tmp}",
                out = inout(zmm_reg) acc,
                tmp = out(zmm_reg) _,
            );
        }
    }
    _mm512_movepi8_mask(acc) as usize
}

I expected to see this happen: Compile successfully.

cargo asm with b14 removed 
asm_bug_minimize::p0:
	.cfi_startproc
	test rsi, rsi
	je .LBB0_1
	push rax
	.cfi_def_cfa_offset 16
	shl rsi, 6
	vpxor xmm14, xmm14, xmm14
	xor eax, eax
	vmovdqa64 zmm0, zmmword ptr [rip + .LCPI0_0]
	vmovdqa64 zmm1, zmmword ptr [rip + .LCPI0_1]
	vmovdqa64 zmm2, zmmword ptr [rip + .LCPI0_2]
	vmovdqa64 zmm3, zmmword ptr [rip + .LCPI0_3]
	vmovdqa64 zmm4, zmmword ptr [rip + .LCPI0_4]
	vmovdqa64 zmm5, zmmword ptr [rip + .LCPI0_5]
	vmovdqa64 zmm6, zmmword ptr [rip + .LCPI0_6]
	vmovdqa64 zmm7, zmmword ptr [rip + .LCPI0_7]
	vmovdqa64 zmm8, zmmword ptr [rip + .LCPI0_8]
	vmovdqa64 zmm9, zmmword ptr [rip + .LCPI0_9]
	vmovdqa64 zmm10, zmmword ptr [rip + .LCPI0_10]
	vmovdqa64 zmm11, zmmword ptr [rip + .LCPI0_11]
	vmovdqa64 zmm12, zmmword ptr [rip + .LCPI0_12]
	vmovdqa64 zmm13, zmmword ptr [rip + .LCPI0_13]
	.p2align	4
.LBB0_3:
	vmovdqa64 zmm15, zmmword ptr [rdi + rax]
	vpcmpeqb k0, zmm15, zmm0
	vpmovm2b zmm16, k0
	vpsubb zmm14, zmm14, zmm16
	vpcmpeqb k0, zmm15, zmm1
	vpmovm2b zmm16, k0
	vpsubb zmm14, zmm14, zmm16
	vpcmpeqb k0, zmm15, zmm2
	vpmovm2b zmm16, k0
	vpsubb zmm14, zmm14, zmm16
	vpcmpeqb k0, zmm15, zmm3
	vpmovm2b zmm16, k0
	vpsubb zmm14, zmm14, zmm16
	vpcmpeqb k0, zmm15, zmm4
	vpmovm2b zmm16, k0
	vpsubb zmm14, zmm14, zmm16
	vpcmpeqb k0, zmm15, zmm5
	vpmovm2b zmm16, k0
	vpsubb zmm14, zmm14, zmm16
	vpcmpeqb k0, zmm15, zmm6
	vpmovm2b zmm16, k0
	vpsubb zmm14, zmm14, zmm16
	vpcmpeqb k0, zmm15, zmm7
	vpmovm2b zmm16, k0
	vpsubb zmm14, zmm14, zmm16
	vpcmpeqb k0, zmm15, zmm8
	vpmovm2b zmm16, k0
	vpsubb zmm14, zmm14, zmm16
	vpcmpeqb k0, zmm15, zmm9
	vpmovm2b zmm16, k0
	vpsubb zmm14, zmm14, zmm16
	vpcmpeqb k0, zmm15, zmm10
	vpmovm2b zmm16, k0
	vpsubb zmm14, zmm14, zmm16
	vpcmpeqb k0, zmm15, zmm11
	vpmovm2b zmm16, k0
	vpsubb zmm14, zmm14, zmm16
	vpcmpeqb k0, zmm15, zmm12
	vpmovm2b zmm16, k0
	vpsubb zmm14, zmm14, zmm16
	vpcmpeqb k0, zmm15, zmm13
	vpmovm2b zmm15, k0
	vpsubb zmm14, zmm14, zmm15
	#APP

	vpaddb zmm15, zmm14, zmm14
	vpaddb zmm14, zmm15, zmm15
	vpaddb zmm14, zmm14, zmm15

	#NO_APP
	add rax, 64
	cmp rsi, rax
	jne .LBB0_3
	vpmovb2m k0, zmm14
	kmovq rax, k0
	add rsp, 8
	.cfi_def_cfa_offset 8
	vzeroupper
	ret
.LBB0_1:
	xor eax, eax
	ret
cargo asm with avx512vl removed 
	.cfi_startproc
	test rsi, rsi
	je .LBB0_1
	push rax
	.cfi_def_cfa_offset 16
	shl rsi, 6
	vpxor xmm15, xmm15, xmm15
	xor eax, eax
	vmovdqa64 zmm18, zmmword ptr [rip + .LCPI0_0]
	vmovdqa64 zmm1, zmmword ptr [rip + .LCPI0_1]
	vmovdqa64 zmm2, zmmword ptr [rip + .LCPI0_2]
	vmovdqa64 zmm3, zmmword ptr [rip + .LCPI0_3]
	vmovdqa64 zmm4, zmmword ptr [rip + .LCPI0_4]
	vmovdqa64 zmm5, zmmword ptr [rip + .LCPI0_5]
	vmovdqa64 zmm6, zmmword ptr [rip + .LCPI0_6]
	vmovdqa64 zmm7, zmmword ptr [rip + .LCPI0_7]
	vmovdqa64 zmm8, zmmword ptr [rip + .LCPI0_8]
	vmovdqa64 zmm9, zmmword ptr [rip + .LCPI0_9]
	vmovdqa64 zmm10, zmmword ptr [rip + .LCPI0_10]
	vmovdqa64 zmm11, zmmword ptr [rip + .LCPI0_11]
	vmovdqa64 zmm12, zmmword ptr [rip + .LCPI0_12]
	vmovdqa64 zmm13, zmmword ptr [rip + .LCPI0_13]
	vmovdqa64 zmm14, zmmword ptr [rip + .LCPI0_14]
	.p2align	4
.LBB0_3:
	vmovdqa64 zmm16, zmmword ptr [rdi + rax]
	vpcmpeqb k0, zmm16, zmm18
	vpmovm2b zmm17, k0
	vpsubb zmm15, zmm15, zmm17
	vpcmpeqb k0, zmm16, zmm1
	vpmovm2b zmm17, k0
	vpsubb zmm15, zmm15, zmm17
	vpcmpeqb k0, zmm16, zmm2
	vpmovm2b zmm17, k0
	vpsubb zmm15, zmm15, zmm17
	vpcmpeqb k0, zmm16, zmm3
	vpmovm2b zmm17, k0
	vpsubb zmm15, zmm15, zmm17
	vpcmpeqb k0, zmm16, zmm4
	vpmovm2b zmm17, k0
	vpsubb zmm15, zmm15, zmm17
	vpcmpeqb k0, zmm16, zmm5
	vpmovm2b zmm17, k0
	vpsubb zmm15, zmm15, zmm17
	vpcmpeqb k0, zmm16, zmm6
	vpmovm2b zmm17, k0
	vpsubb zmm15, zmm15, zmm17
	vpcmpeqb k0, zmm16, zmm7
	vpmovm2b zmm17, k0
	vpsubb zmm15, zmm15, zmm17
	vpcmpeqb k0, zmm16, zmm8
	vpmovm2b zmm17, k0
	vpsubb zmm15, zmm15, zmm17
	vpcmpeqb k0, zmm16, zmm9
	vpmovm2b zmm17, k0
	vpsubb zmm15, zmm15, zmm17
	vpcmpeqb k0, zmm16, zmm10
	vpmovm2b zmm17, k0
	vpsubb zmm15, zmm15, zmm17
	vpcmpeqb k0, zmm16, zmm11
	vpmovm2b zmm17, k0
	vpsubb zmm15, zmm15, zmm17
	vpcmpeqb k0, zmm16, zmm12
	vpmovm2b zmm17, k0
	vpsubb zmm15, zmm15, zmm17
	vpcmpeqb k0, zmm16, zmm13
	vpmovm2b zmm17, k0
	vpsubb zmm15, zmm15, zmm17
	vpcmpeqb k0, zmm16, zmm14
	vpmovm2b zmm16, k0
	vpsubb zmm15, zmm15, zmm16
	#APP

	vpaddb zmm0, zmm15, zmm15
	vpaddb zmm15, zmm0, zmm0
	vpaddb zmm15, zmm15, zmm0

	#NO_APP
	add rax, 64
	cmp rsi, rax
	jne .LBB0_3
	vpmovb2m k0, zmm15
	kmovq rax, k0
	add rsp, 8
	.cfi_def_cfa_offset 8
	vzeroupper
	ret
.LBB0_1:
	xor eax, eax
	ret

Instead, this happened: Compile error

1.94 and earlier (LLVM 21), nightly-2025-05-09 (LLVM 20) 
error: invalid operand for instruction
  --> <source>:41:18
   |
41 |                 "vpaddb {tmp}, {out}, {out}",
   |                  ^^^^^^^^^^^^^^^^^^^^^^^^^^
   |
note: instantiated into assembly here
  --> <inline asm>:2:2
   |
 2 |     vpaddb r31b, zmm15, zmm15
   |     ^

error: invalid operand for instruction
  --> <source>:42:18
   |
42 |                 "vpaddb {out}, {tmp}, {tmp}",
   |                  ^^^^^^^^^^^^^^^^^^^^^^^^^^
   |
note: instantiated into assembly here
  --> <inline asm>:3:1
   |
 3 | vpaddb zmm15, r31b, r31b
   | ^

error: invalid operand for instruction
  --> <source>:43:18
   |
43 |                 "vpaddb {out}, {out}, {tmp}",
   |                  ^^^^^^^^^^^^^^^^^^^^^^^^^^
   |
note: instantiated into assembly here
  --> <inline asm>:4:1
   |
 4 | vpaddb zmm15, zmm15, r31b
   | ^

error: aborting due to 3 previous errors

Compiler returned: 1
1.95 and later including nightly-2026-07-15 (LLVM 22), nightly-2025-02-17 (LLVM 19) 
error: invalid operand for instruction
  --> <source>:41:18
   |
41 |                 "vpaddb {tmp}, {out}, {out}",
   |                  ^^^^^^^^^^^^^^^^^^^^^^^^^^
   |
note: instantiated into assembly here
  --> <inline asm>:2:2
   |
 2 |     vpaddb R19BH, zmm15, zmm15
   |     ^

error: invalid operand for instruction
  --> <source>:42:18
   |
42 |                 "vpaddb {out}, {tmp}, {tmp}",
   |                  ^^^^^^^^^^^^^^^^^^^^^^^^^^
   |
note: instantiated into assembly here
  --> <inline asm>:3:1
   |
 3 | vpaddb zmm15, R19BH, R19BH
   | ^

error: aborting due to 2 previous errors

Compiler returned: 1

However, if you extend the pattern to b29, or remove the avx512vl target feature (which is technically unused in this example), or change the inline asm block to

unsafe {
    std::arch::asm!(
        "vpaddb {tmp}, {out}, {out}",
        "vpaddb {out}, {tmp}, {tmp}",
        "vpaddb {tmp}, {out}, {tmp}",
        out = inout(zmm_reg) acc => _,
        tmp = out(zmm_reg) acc,
    );
}

it works.

Meta

rustc --version --verbose:

rustc 1.99.0-nightly (d0babd8b6 2026-07-15)
binary: rustc
commit-hash: d0babd8b6b05ef9bb65d42f928cef4129d64cf65
commit-date: 2026-07-15
host: x86_64-unknown-linux-gnu
release: 1.99.0-nightly
LLVM version: 22.1.8

Contributor guide

Open the contributing guide

First steps

  1. Read the whole issue, then the project's contributing guide.
  2. Comment on the issue to say you are picking it up — it saves two people doing the same work.
  3. Fork the repository and make your change on a branch.
  4. Open a pull request that references the issue number.

Research direction

Start by compiling the provided x86-64 Rust reproducer with the listed target-feature combinations and inspect the inline asm register allocation and generated assembly. Compare the failing out(zmm_reg) _ case with the working alternatives and the cargo asm output shown in the issue. Done means the reproducer compiles without invalid operands and the discarded vector-output cases exhibit the expected assembly behavior, including the reported vzeroupper issue.

Written by the indexing model from the issue text.

Assessment

Tech stack
rust
Domain
compilers
Issue type
Bug
Difficulty
4/5
Estimated time
3-5 days
Activity status
Quiet
Clarity
Mostly clear
Newbie friendliness
45/100

Get new issues in your inbox

A short digest of beginner-friendly GitHub issues.