[SLP] Loop Aware cost model causes regression in SPEC2017 508.namd_r

@karouzakisp is already working on this.

Since Jul 15, 2026.

This PR https://github.com/llvm/llvm-project/pull/150450 caused a 3.32% regression in SPEC 2017 508.namd_r on a Neoverse V2 machine.

The changes causing the regression seem to be relatively flat, spread over several similar functions. However the crux of the issue seems to be that reduction chains which are spread across multiple sibling loops that were previously vectorized are now failing to do so, due to new logic which chooses to avoid proceeding with Trees that span multiple sibling loops when these loop's BTCs are represented by different SCEVs.

This issue is illustrated by the following example

`:`

```llvm
define dso_local void @f1(ptr noundef readonly captures(none) %arr, ptr noundef writeonly captures(none) %out) {
entry:
br label %outer.header

outer.header: ; preds = %outer.latch, %entry
%outer.iv = phi i32 [ 0, %entry ], [ %outer.next, %outer.latch ]
%acc1.outer = phi double [ 0.000000e+00, %entry ], [ %acc1.after.second.lcssa, %outer.latch ]
%acc0.outer = phi double [ 0.000000e+00, %entry ], [ %acc0.after.second.lcssa, %outer.latch ]
br label %first.header

first.header: ; preds = %first.header, %outer.header
%i.first = phi i64 [ 0, %outer.header ], [ %i.first.next, %first.header ]
%acc1.first = phi double [ %acc1.outer, %outer.header ], [ %acc1.first.next, %first.header ]
%acc0.first = phi double [ %acc0.outer, %outer.header ], [ %acc0.first.next, %first.header ]
%p0 = getelementptr inbounds nuw double, ptr %arr, i64 %i.first
%v0 = load double, ptr %p0, align 8
%v0a = fadd double %v0, 2.500000e-01
%v0b = fadd double %v0a, 5.000000e-01
%acc0.first.next = fadd double %v0b, %acc0.first
%p1 = getelementptr inbounds nuw i8, ptr %p0, i64 8
%v1 = load double, ptr %p1, align 8
%v1a = fadd double %v1, 2.500000e-01
%v1b = fadd double %v1a, 5.000000e-01
%acc1.first.next = fadd double %v1b, %acc1.first
%i.first.next = add nuw nsw i64 %i.first, 1
%first.cont = icmp ult i64 %i.first.next, 32
br i1 %first.cont, label %first.header, label %second.header, !llvm.loop !0

second.header: ; preds = %first.header, %second.header
%i.second = phi i64 [ %i.second.next, %second.header ], [ 0, %first.header ]
%acc1.second = phi double [ %acc1.after.second, %second.header ], [ %acc1.first.next, %first.header ]
%acc0.second = phi double [ %acc0.after.second, %second.header ], [ %acc0.first.next, %first.header ]
%q0 = getelementptr inbounds nuw double, ptr %arr, i64 %i.second
%w0 = load double, ptr %q0, align 8
%w0.bias = fadd double %w0, 1.000000e+00
%w0.acc = fadd double %w0.bias, %w0.bias
%acc0.after.second = fadd double %w0.acc, %acc0.second
%q1 = getelementptr inbounds nuw i8, ptr %q0, i64 8
%w1 = load double, ptr %q1, align 8
%w1.bias = fadd double %w1, 1.000000e+00
%w1.acc = fadd double %w1.bias, %w1.bias
%acc1.after.second = fadd double %w1.acc, %acc1.second
%i.second.next = add nuw nsw i64 %i.second, 1
%second.cont = icmp ult i64 %i.second.next, 2
br i1 %second.cont, label %second.header, label %outer.latch, !llvm.loop !3

outer.latch: ; preds = %second.header
%acc0.after.second.lcssa = phi double [ %acc0.after.second, %second.header ]
%acc1.after.second.lcssa = phi double [ %acc1.after.second, %second.header ]
%outer.next = add nuw nsw i32 %outer.iv, 1
%outer.cont = icmp ult i32 %outer.next, 2
br i1 %outer.cont, label %outer.header, label %exit, !llvm.loop !4

exit: ; preds = %outer.latch
%acc0.after.second.lcssa.lcssa = phi double [ %acc0.after.second.lcssa, %outer.latch ]
%acc1.after.second.lcssa.lcssa = phi double [ %acc1.after.second.lcssa, %outer.latch ]
%sum = fadd double %acc0.after.second.lcssa.lcssa, %acc1.after.second.lcssa.lcssa
store double %sum, ptr %out, align 8
ret void
}

attributes #0 = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) }

!0 = distinct !{!0, !1, !2}
!1 = !{!"llvm.loop.mustprogress"}
!2 = !{!"llvm.loop.unroll.disable"}
!3 = distinct !{!3, !1, !2}
!4 = distinct !{!4, !1, !2}
```

Running SLP on this small example (`opt -S -mtriple=aarch64-linux-gnu -passes=slp-vectorizer repro.ll`) produces the following IR:

```llvm
define dso_local void @f1(ptr noundef readonly captures(none) %arr, ptr noundef writeonly captures(none) %out) {
entry:
br label %outer.header

outer.header: ; preds = %outer.latch, %entry
%outer.iv = phi i32 [ 0, %entry ], [ %outer.next, %outer.latch ]
%0 = phi <2 x double> [ zeroinitializer, %entry ], [ %9, %outer.latch ]
br label %first.header

first.header: ; preds = %first.header, %outer.header
%i.first = phi i64 [ 0, %outer.header ], [ %i.first.next, %first.header ]
%1 = phi <2 x double> [ %0, %outer.header ], [ %5, %first.header ]
%p0 = getelementptr inbounds nuw double, ptr %arr, i64 %i.first
%2 = load <2 x double>, ptr %p0, align 8
%3 = fadd <2 x double> %2, splat (double 2.500000e-01)
%4 = fadd <2 x double> %3, splat (double 5.000000e-01)
%5 = fadd <2 x double> %4, %1
%i.first.next = add nuw nsw i64 %i.first, 1
%first.cont = icmp ult i64 %i.first.next, 32
%6 = extractelement <2 x double> %5, i64 0
%7 = extractelement <2 x double> %5, i64 1
br i1 %first.cont, label %first.header, label %second.header, !llvm.loop !0

second.header: ; preds = %second.header, %first.header
%i.second = phi i64 [ %i.second.next, %second.header ], [ 0, %first.header ]
%acc1.second = phi double [ %acc1.after.second, %second.header ], [ %7, %first.header ]
%acc0.second = phi double [ %acc0.after.second, %second.header ], [ %6, %first.header ]
%q0 = getelementptr inbounds nuw double, ptr %arr, i64 %i.second
%w0 = load double, ptr %q0, align 8
%w0.bias = fadd double %w0, 1.000000e+00
%w0.acc = fadd double %w0.bias, %w0.bias
%acc0.after.second = fadd double %w0.acc, %acc0.second
%q1 = getelementptr inbounds nuw i8, ptr %q0, i64 8
%w1 = load double, ptr %q1, align 8
%w1.bias = fadd double %w1, 1.000000e+00
%w1.acc = fadd double %w1.bias, %w1.bias
%acc1.after.second = fadd double %w1.acc, %acc1.second
%i.second.next = add nuw nsw i64 %i.second, 1
%second.cont = icmp ult i64 %i.second.next, 2
br i1 %second.cont, label %second.header, label %outer.latch, !llvm.loop !3

outer.latch: ; preds = %second.header
%acc0.after.second.lcssa = phi double [ %acc0.after.second, %second.header ]
%acc1.after.second.lcssa = phi double [ %acc1.after.second, %second.header ]
%outer.next = add nuw nsw i32 %outer.iv, 1
%outer.cont = icmp ult i32 %outer.next, 2
%8 = insertelement <2 x double> poison, double %acc0.after.second.lcssa, i64 0
%9 = insertelement <2 x double> %8, double %acc1.after.second.lcssa, i64 1
br i1 %outer.cont, label %outer.header, label %exit, !llvm.loop !4

exit: ; preds = %outer.latch
%acc0.after.second.lcssa.lcssa = phi double [ %acc0.after.second.lcssa, %outer.latch ]
%acc1.after.second.lcssa.lcssa = phi double [ %acc1.after.second.lcssa, %outer.latch ]
%sum = fadd double %acc0.after.second.lcssa.lcssa, %acc1.after.second.lcssa.lcssa
store double %sum, ptr %out, align 8
ret void
}

!0 = distinct !{!0, !1, !2}
!1 = !{!"llvm.loop.mustprogress"}
!2 = !{!"llvm.loop.unroll.disable"}
!3 = distinct !{!3, !1, !2}
!4 = distinct !{!4, !1, !2}
```

SLP vectorizes the loads and adds in the first loop, but not in the second loop.

From the SLP debug output, we can see that when we are calculating the cost of the Tree rooted at the two accumulator exit phis in BB `exit` (`{%acc0.after.second.lcssa.lcssa, %acc1.after.second.lcssa.lcssa}`) we hit the mismatching BTC SCEV condition when we encounter the accumulators from the "first" loop (`{%acc0.first.next, %acc1.first.next}`) which feed the "second" loop's header phis (`{%acc0.second, %acc1.second}`) and we mark this TE as a Gather.

When costing this Gather, we give it a base cost of 2 which is then multiplied by the loop-aware scaling to a cost of 128:

```
SLP: Adding cost 128 for bundle Idx: 10, n=2 [ %acc0.first.next = fadd double %v0b, %acc0.first, ..].
```

which ends up far outweighing the savings of the vectorizable TEs in the Tree and we decide not to vectorize this Tree.

Looking into what is going on here, in `getScaleToLoopIterations` we follow the uses of the Gather back down the Tree until we reach the TE corresponding to the the second loop's header phis, and then we select the incoming block as the context from which to scale
https://github.com/llvm/llvm-project/blob/41388b1d74d4fa973db029de6ef89c03149bcb19/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp#L16751-L16762

this would be a sensible thing to do if we had a loop in Loop Simplify Form: the incoming block would be the loop's preheader, and it makes sense that the gather of these loop-invariant (in second-loop) scalars from the first-loop would be paid for "once-per-entry" to second-loop.

However as the loop has no preheader, and the incoming block is inside the first-loop, we scale the base cost of 2 by `outer-loop TC (2) * first-loop TC (32)`, which is why we get such a large cost. Now this obviously doesn't really make any sense here; the if we are gathering scalars from the first-loop for use in the second-loop we wouldn't pay that per iteration of the first-loop, it would be sunk.

(In reality, in this particular case these scalars are themselves Vectorized when we subsequently look at the Tree in first-loop, so the Gather TE is actually essentially free).

One approach to solving this could be to extend the logic in `getScaleToLoopIterations` to improve the selection of the Basic Block context, for example to recognize this kind of sibling-loop bridging Phi and using a context outside of both loops (?Common Parent).

However, the underlying issue is really that the loop is not in Loop Simplify Form and does not have a separate preheader: this is representative of input IR to SLP which has been aggressively simplified by SimplifyCFG.

Running LoopSimplifyPass before SLP on our example (`opt -S -mtriple=aarch64-linux-gnu -passes=loop-simplify,slp-vectorizer repro.ll`) produces the following IR:

```llvm
define dso_local void @f1(ptr noundef readonly captures(none) %arr, ptr noundef writeonly captures(none) %out) {
entry:
br label %outer.header

outer.header: ; preds = %outer.latch, %entry
%outer.iv = phi i32 [ 0, %entry ], [ %outer.next, %outer.latch ]
%0 = phi <2 x double> [ zeroinitializer, %entry ], [ %11, %outer.latch ]
br label %first.header

first.header: ; preds = %first.header, %outer.header
%i.first = phi i64 [ 0, %outer.header ], [ %i.first.next, %first.header ]
%1 = phi <2 x double> [ %0, %outer.header ], [ %5, %first.header ]
%p0 = getelementptr inbounds nuw double, ptr %arr, i64 %i.first
%2 = load <2 x double>, ptr %p0, align 8
%3 = fadd <2 x double> %2, splat (double 2.500000e-01)
%4 = fadd <2 x double> %3, splat (double 5.000000e-01)
%5 = fadd <2 x double> %4, %1
%i.first.next = add nuw nsw i64 %i.first, 1
%first.cont = icmp ult i64 %i.first.next, 32
br i1 %first.cont, label %first.header, label %second.header.preheader, !llvm.loop !0

second.header.preheader: ; preds = %first.header
br label %second.header

second.header: ; preds = %second.header.preheader, %second.header
%i.second = phi i64 [ %i.second.next, %second.header ], [ 0, %second.header.preheader ]
%6 = phi <2 x double> [ %10, %second.header ], [ %5, %second.header.preheader ]
%q0 = getelementptr inbounds nuw double, ptr %arr, i64 %i.second
%7 = load <2 x double>, ptr %q0, align 8
%8 = fadd <2 x double> %7, splat (double 1.000000e+00)
%9 = fadd <2 x double> %8, %8
%10 = fadd <2 x double> %9, %6
%i.second.next = add nuw nsw i64 %i.second, 1
%second.cont = icmp ult i64 %i.second.next, 2
br i1 %second.cont, label %second.header, label %outer.latch, !llvm.loop !3

outer.latch: ; preds = %second.header
%11 = phi <2 x double> [ %10, %second.header ]
%outer.next = add nuw nsw i32 %outer.iv, 1
%outer.cont = icmp ult i32 %outer.next, 2
br i1 %outer.cont, label %outer.header, label %exit, !llvm.loop !4

exit: ; preds = %outer.latch
%12 = phi <2 x double> [ %11, %outer.latch ]
%13 = extractelement <2 x double> %12, i64 0
%14 = extractelement <2 x double> %12, i64 1
%sum = fadd double %13, %14
store double %sum, ptr %out, align 8
ret void
}

!0 = distinct !{!0, !1, !2}
!1 = !{!"llvm.loop.mustprogress"}
!2 = !{!"llvm.loop.unroll.disable"}
!3 = distinct !{!3, !1, !2}
!4 = distinct !{!4, !1, !2}
```

You can see that we then do vectorize all of the FP operations across both loops. So perhaps another approach here would be to add a LoopSimplifyPass() run before SLP in the default pipeline.